mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-04 02:04:03 +00:00
## What changed? Extended the existing OpenTelemetry (OTEL) functionality with a "debug mode" to emit the gRPC request and response payloads. ## Why? This is the first step in a larger effort to improve _local_ observability of the Server. There is a need to be able to observe important events - such as gRPC calls here - in greater detail during development. Work on _consumers_ of that data is out of scope for this PR. ## How did you test it? (1) `docker run \ -p 127.0.0.1:4317:4317 \ -p 127.0.0.1:55679:55679 \ otel/opentelemetry-collector:0.106.1 \ 2>&1 | tee collector-output.txt` (2) `OTEL_TRACES_EXPORTER=otlp OTEL_EXPORTER_OTLP_TRACES_INSECURE=true TEMPORAL_OTEL_DEBUG=true make start` (3) observed gRPC traces in the output file, for example: ``` Span #0 Trace ID : 305f55932d35b89d376e2f61b0a0d315 Parent ID : 83189aaab963c7dc ID : d1d0420413bc9880 Name : temporal.server.api.historyservice.v1.HistoryService/StartWorkflowExecution Kind : Server Start time : 2024-08-14 00:16:32.912426 +0000 UTC End time : 2024-08-14 00:16:32.915583542 +0000 UTC Status code : Unset Status message : Attributes: -> rpc.system: Str(grpc) -> rpc.service: Str(temporal.server.api.historyservice.v1.HistoryService) -> rpc.method: Str(StartWorkflowExecution) -> net.sock.peer.addr: Str(127.0.0.1) -> net.sock.peer.port: Int(56016) -> rpc.request.payload: Str({"namespaceId":"32049b68-7872-4094-8e63-d0dd59896a83","startRequest":{"namespace":"temporal-system","workflowId":"temporal-sys-tq-scanner","workflowType":{"name":"temporal-sys-tq-scanner-workflow"},"taskQueue":{"name":"temporal-sys-tq-scanner-taskqueue-0","kind":"TASK_QUEUE_KIND_NORMAL"},"workflowExecutionTimeout":"0s","workflowRunTimeout":"432000s","workflowTaskTimeout":"120s","identity":"[email protected]@","requestId":"3d88c924-e9b3-4415-a3e5-8aa207849550","workflowIdReusePolicy":"WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE","workflowIdConflictPolicy":"WORKFLOW_ID_CONFLICT_POLICY_FAIL","cronSchedule":"0 */12 * * *","header":{}},"attempt":1,"continueAsNewInitiator":"CONTINUE_AS_NEW_INITIATOR_CRON_SCHEDULE","firstWorkflowTaskBackoff":"42208s"}) -> rpc.request.type: Str(StartWorkflowExecutionRequest) -> rpc.response.payload: Str({"runId":"33962239-982f-4fa3-91d3-db2f6e796883","clock":{"shardId":1,"clock":"1048585","clusterId":"1"},"started":true}) -> rpc.response.type: Str(StartWorkflowExecutionResponse) -> rpc.grpc.status_code: Int(0) ``` ## Potential risks The debug functionality is guarded through an environment variable. ## Is hotfix candidate? No
115 lines
4.0 KiB
Go
115 lines
4.0 KiB
Go
// The MIT License
|
|
//
|
|
// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved.
|
|
//
|
|
// Copyright (c) 2020 Uber Technologies, Inc.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
// THE SOFTWARE.
|
|
|
|
package telemetry
|
|
|
|
import (
|
|
"context"
|
|
|
|
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
"go.opentelemetry.io/otel/trace"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/protobuf/encoding/protojson"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
type (
|
|
// ServerTraceInterceptor gives a named type to the
|
|
// grpc.UnaryServerInterceptor implementation provided by otelgrpc
|
|
ServerTraceInterceptor grpc.UnaryServerInterceptor
|
|
|
|
// ClientTraceInterceptor gives a named type to the
|
|
// grpc.UnaryClientInterceptor implementation provided by otelgrpc
|
|
ClientTraceInterceptor grpc.UnaryClientInterceptor
|
|
)
|
|
|
|
// NewServerTraceInterceptor creates a new gRPC server interceptor that tracks
|
|
// each request with an encapsulating span using the provided TracerProvider and
|
|
// TextMapPropagator.
|
|
func NewServerTraceInterceptor(
|
|
tp trace.TracerProvider,
|
|
tmp propagation.TextMapPropagator,
|
|
) ServerTraceInterceptor {
|
|
//nolint:staticcheck
|
|
otelInterceptor := otelgrpc.UnaryServerInterceptor(
|
|
otelgrpc.WithPropagators(tmp),
|
|
otelgrpc.WithTracerProvider(tp),
|
|
)
|
|
|
|
if debugMode() {
|
|
return func(
|
|
ctx context.Context,
|
|
req interface{},
|
|
info *grpc.UnaryServerInfo,
|
|
handler grpc.UnaryHandler,
|
|
) (any, error) {
|
|
return otelInterceptor(ctx, req, info, debugHandler(handler))
|
|
}
|
|
}
|
|
|
|
return ServerTraceInterceptor(otelInterceptor)
|
|
}
|
|
|
|
func debugHandler(origHandler grpc.UnaryHandler) grpc.UnaryHandler {
|
|
return func(ctx context.Context, req any) (resp any, err error) {
|
|
resp, err = origHandler(ctx, req)
|
|
span := trace.SpanFromContext(ctx)
|
|
if span.IsRecording() {
|
|
//revive:disable-next-line:unchecked-type-assertion
|
|
reqMsg := req.(proto.Message)
|
|
payload, _ := protojson.Marshal(reqMsg)
|
|
msgType := string(proto.MessageName(reqMsg).Name())
|
|
span.SetAttributes(attribute.Key("rpc.request.payload").String(string(payload)))
|
|
span.SetAttributes(attribute.Key("rpc.request.type").String(msgType))
|
|
|
|
if err == nil {
|
|
//revive:disable-next-line:unchecked-type-assertion
|
|
respMsg := resp.(proto.Message)
|
|
payload, _ = protojson.Marshal(respMsg)
|
|
msgType = string(proto.MessageName(respMsg).Name())
|
|
span.SetAttributes(attribute.Key("rpc.response.payload").String(string(payload)))
|
|
span.SetAttributes(attribute.Key("rpc.response.type").String(msgType))
|
|
}
|
|
}
|
|
return
|
|
}
|
|
}
|
|
|
|
// NewClientTraceInterceptor creates a new gRPC client interceptor that tracks
|
|
// each request with an encapsulating span using the provided TracerProvider and
|
|
// TextMapPropagator.
|
|
func NewClientTraceInterceptor(
|
|
tp trace.TracerProvider,
|
|
tmp propagation.TextMapPropagator,
|
|
) ClientTraceInterceptor {
|
|
return ClientTraceInterceptor(
|
|
otelgrpc.UnaryClientInterceptor(
|
|
otelgrpc.WithPropagators(tmp),
|
|
otelgrpc.WithTracerProvider(tp),
|
|
),
|
|
)
|
|
}
|