Files
Stephan BehnkeandLina Jodoin a8e58e60df gRPC debug telemetry (#6402)
## What changed?

Extended the existing OpenTelemetry (OTEL) functionality with a "debug
mode" to emit the gRPC request and response payloads.

## Why?

This is the first step in a larger effort to improve _local_
observability of the Server. There is a need to be able to observe
important events - such as gRPC calls here - in greater detail during
development.

Work on _consumers_ of that data is out of scope for this PR.

## How did you test it?

(1) `docker run \
  -p 127.0.0.1:4317:4317 \
  -p 127.0.0.1:55679:55679 \
  otel/opentelemetry-collector:0.106.1 \
  2>&1 | tee collector-output.txt`
(2) `OTEL_TRACES_EXPORTER=otlp OTEL_EXPORTER_OTLP_TRACES_INSECURE=true
TEMPORAL_OTEL_DEBUG=true make start`
(3) observed gRPC traces in the output file, for example:

```
Span #0
    Trace ID       : 305f55932d35b89d376e2f61b0a0d315
    Parent ID      : 83189aaab963c7dc
    ID             : d1d0420413bc9880
    Name           : temporal.server.api.historyservice.v1.HistoryService/StartWorkflowExecution
    Kind           : Server
    Start time     : 2024-08-14 00:16:32.912426 +0000 UTC
    End time       : 2024-08-14 00:16:32.915583542 +0000 UTC
    Status code    : Unset
    Status message : 
Attributes:
     -> rpc.system: Str(grpc)
     -> rpc.service: Str(temporal.server.api.historyservice.v1.HistoryService)
     -> rpc.method: Str(StartWorkflowExecution)
     -> net.sock.peer.addr: Str(127.0.0.1)
     -> net.sock.peer.port: Int(56016)
     -> rpc.request.payload: Str({"namespaceId":"32049b68-7872-4094-8e63-d0dd59896a83","startRequest":{"namespace":"temporal-system","workflowId":"temporal-sys-tq-scanner","workflowType":{"name":"temporal-sys-tq-scanner-workflow"},"taskQueue":{"name":"temporal-sys-tq-scanner-taskqueue-0","kind":"TASK_QUEUE_KIND_NORMAL"},"workflowExecutionTimeout":"0s","workflowRunTimeout":"432000s","workflowTaskTimeout":"120s","identity":"[email protected]@","requestId":"3d88c924-e9b3-4415-a3e5-8aa207849550","workflowIdReusePolicy":"WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE","workflowIdConflictPolicy":"WORKFLOW_ID_CONFLICT_POLICY_FAIL","cronSchedule":"0 */12 * * *","header":{}},"attempt":1,"continueAsNewInitiator":"CONTINUE_AS_NEW_INITIATOR_CRON_SCHEDULE","firstWorkflowTaskBackoff":"42208s"})
     -> rpc.request.type: Str(StartWorkflowExecutionRequest)
     -> rpc.response.payload: Str({"runId":"33962239-982f-4fa3-91d3-db2f6e796883","clock":{"shardId":1,"clock":"1048585","clusterId":"1"},"started":true})
     -> rpc.response.type: Str(StartWorkflowExecutionResponse)
     -> rpc.grpc.status_code: Int(0)
```

## Potential risks

The debug functionality is guarded through an environment variable. 

## Is hotfix candidate?

No
2024-08-30 12:54:11 -07:00

115 lines
4.0 KiB
Go

// The MIT License
//
// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved.
//
// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package telemetry
import (
"context"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
)
type (
// ServerTraceInterceptor gives a named type to the
// grpc.UnaryServerInterceptor implementation provided by otelgrpc
ServerTraceInterceptor grpc.UnaryServerInterceptor
// ClientTraceInterceptor gives a named type to the
// grpc.UnaryClientInterceptor implementation provided by otelgrpc
ClientTraceInterceptor grpc.UnaryClientInterceptor
)
// NewServerTraceInterceptor creates a new gRPC server interceptor that tracks
// each request with an encapsulating span using the provided TracerProvider and
// TextMapPropagator.
func NewServerTraceInterceptor(
tp trace.TracerProvider,
tmp propagation.TextMapPropagator,
) ServerTraceInterceptor {
//nolint:staticcheck
otelInterceptor := otelgrpc.UnaryServerInterceptor(
otelgrpc.WithPropagators(tmp),
otelgrpc.WithTracerProvider(tp),
)
if debugMode() {
return func(
ctx context.Context,
req interface{},
info *grpc.UnaryServerInfo,
handler grpc.UnaryHandler,
) (any, error) {
return otelInterceptor(ctx, req, info, debugHandler(handler))
}
}
return ServerTraceInterceptor(otelInterceptor)
}
func debugHandler(origHandler grpc.UnaryHandler) grpc.UnaryHandler {
return func(ctx context.Context, req any) (resp any, err error) {
resp, err = origHandler(ctx, req)
span := trace.SpanFromContext(ctx)
if span.IsRecording() {
//revive:disable-next-line:unchecked-type-assertion
reqMsg := req.(proto.Message)
payload, _ := protojson.Marshal(reqMsg)
msgType := string(proto.MessageName(reqMsg).Name())
span.SetAttributes(attribute.Key("rpc.request.payload").String(string(payload)))
span.SetAttributes(attribute.Key("rpc.request.type").String(msgType))
if err == nil {
//revive:disable-next-line:unchecked-type-assertion
respMsg := resp.(proto.Message)
payload, _ = protojson.Marshal(respMsg)
msgType = string(proto.MessageName(respMsg).Name())
span.SetAttributes(attribute.Key("rpc.response.payload").String(string(payload)))
span.SetAttributes(attribute.Key("rpc.response.type").String(msgType))
}
}
return
}
}
// NewClientTraceInterceptor creates a new gRPC client interceptor that tracks
// each request with an encapsulating span using the provided TracerProvider and
// TextMapPropagator.
func NewClientTraceInterceptor(
tp trace.TracerProvider,
tmp propagation.TextMapPropagator,
) ClientTraceInterceptor {
return ClientTraceInterceptor(
otelgrpc.UnaryClientInterceptor(
otelgrpc.WithPropagators(tmp),
otelgrpc.WithTracerProvider(tp),
),
)
}