mirror of
https://github.com/hanzoai/tasks.git
synced 2026-11-03 03:56:04 +00:00
Yichao YangandGitHub
a56af8c3e7
## What changed? - Add per namespace rate limiter on history service ## Why? - Noisy neighbor protection. Provide know for preventing one namespace from consuming all available history host rps. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s)
114 lines
3.7 KiB
Go
114 lines
3.7 KiB
Go
package history
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"go.temporal.io/api/serviceerror"
|
|
"go.temporal.io/server/api/historyservice/v1"
|
|
"go.temporal.io/server/common/headers"
|
|
"go.temporal.io/server/common/log"
|
|
"go.temporal.io/server/common/log/tag"
|
|
"go.temporal.io/server/common/metrics"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
var _ historyservice.HistoryServiceClient = (*metricClient)(nil)
|
|
|
|
type metricClient struct {
|
|
client historyservice.HistoryServiceClient
|
|
metricsHandler metrics.Handler
|
|
logger log.Logger
|
|
throttledLogger log.Logger
|
|
}
|
|
|
|
// NewMetricClient creates a new instance of historyservice.HistoryServiceClient that emits metrics
|
|
func NewMetricClient(
|
|
client historyservice.HistoryServiceClient,
|
|
metricsHandler metrics.Handler,
|
|
logger log.Logger,
|
|
throttledLogger log.Logger,
|
|
) historyservice.HistoryServiceClient {
|
|
return &metricClient{
|
|
client: client,
|
|
metricsHandler: metricsHandler,
|
|
logger: logger,
|
|
throttledLogger: throttledLogger,
|
|
}
|
|
}
|
|
|
|
func (c *metricClient) StreamWorkflowReplicationMessages(
|
|
ctx context.Context,
|
|
opts ...grpc.CallOption,
|
|
) (_ historyservice.HistoryService_StreamWorkflowReplicationMessagesClient, retError error) {
|
|
|
|
metricsHandler, startTime := c.startMetricsRecording(ctx, metrics.HistoryClientStreamWorkflowReplicationMessagesScope)
|
|
defer func() {
|
|
c.finishMetricsRecording(metricsHandler, startTime, retError)
|
|
}()
|
|
|
|
return c.client.StreamWorkflowReplicationMessages(ctx, opts...)
|
|
}
|
|
|
|
func (c *metricClient) StartNexusOperation(
|
|
ctx context.Context,
|
|
request *historyservice.StartNexusOperationRequest,
|
|
opts ...grpc.CallOption,
|
|
) (_ *historyservice.StartNexusOperationResponse, retError error) {
|
|
op := "HistoryClientStartNexusOperation_" + request.GetRequest().GetService() + "_" + request.GetRequest().GetOperation()
|
|
metricsHandler, startTime := c.startMetricsRecording(ctx, op)
|
|
defer func() {
|
|
c.finishMetricsRecording(metricsHandler, startTime, retError)
|
|
}()
|
|
|
|
return c.client.StartNexusOperation(ctx, request, opts...)
|
|
}
|
|
|
|
func (c *metricClient) CancelNexusOperation(
|
|
ctx context.Context,
|
|
request *historyservice.CancelNexusOperationRequest,
|
|
opts ...grpc.CallOption,
|
|
) (_ *historyservice.CancelNexusOperationResponse, retError error) {
|
|
op := "HistoryClientCancelNexusOperation_" + request.GetRequest().GetService() + "_" + request.GetRequest().GetOperation()
|
|
metricsHandler, startTime := c.startMetricsRecording(ctx, op)
|
|
defer func() {
|
|
c.finishMetricsRecording(metricsHandler, startTime, retError)
|
|
}()
|
|
|
|
return c.client.CancelNexusOperation(ctx, request, opts...)
|
|
}
|
|
|
|
func (c *metricClient) startMetricsRecording(
|
|
ctx context.Context,
|
|
operation string,
|
|
) (metrics.Handler, time.Time) {
|
|
caller := headers.GetCallerInfo(ctx).CallerName
|
|
metricsHandler := c.metricsHandler.WithTags(metrics.OperationTag(operation), metrics.NamespaceTag(caller), metrics.ServiceRoleTag(metrics.HistoryRoleTagValue))
|
|
metrics.ClientRequests.With(metricsHandler).Record(1)
|
|
return metricsHandler, time.Now().UTC()
|
|
}
|
|
|
|
func (c *metricClient) finishMetricsRecording(
|
|
metricsHandler metrics.Handler,
|
|
startTime time.Time,
|
|
err error,
|
|
) {
|
|
if err != nil {
|
|
switch err.(type) {
|
|
case *serviceerror.Canceled,
|
|
*serviceerror.DeadlineExceeded,
|
|
*serviceerror.NotFound,
|
|
*serviceerror.QueryFailed,
|
|
*serviceerror.NamespaceNotFound,
|
|
*serviceerror.WorkflowNotReady,
|
|
*serviceerror.WorkflowExecutionAlreadyStarted,
|
|
*serviceerror.ResourceExhausted:
|
|
// noop - not interest and too many logs
|
|
default:
|
|
c.throttledLogger.Info("history client encountered error", tag.Error(err), tag.ServiceErrorType(err))
|
|
}
|
|
metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err))
|
|
}
|
|
metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime))
|
|
}
|