mirror of
https://github.com/hanzoai/tasks.git
synced 2026-10-04 02:04:03 +00:00
Yichao YangandGitHub
198f4d9c5f
## What changed? - Add per namespace rate limiter for matching service - Same as https://github.com/temporalio/temporal/pull/9884 but for matching. ## Why? - Noisy neighbor protection. Provide knob for preventing one namespace from consuming all available matching host rps. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s)
83 lines
2.5 KiB
Go
83 lines
2.5 KiB
Go
package admin
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"go.temporal.io/api/serviceerror"
|
|
"go.temporal.io/server/api/adminservice/v1"
|
|
"go.temporal.io/server/common/headers"
|
|
"go.temporal.io/server/common/log"
|
|
"go.temporal.io/server/common/log/tag"
|
|
"go.temporal.io/server/common/metrics"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
var _ adminservice.AdminServiceClient = (*metricClient)(nil)
|
|
|
|
type metricClient struct {
|
|
client adminservice.AdminServiceClient
|
|
metricsHandler metrics.Handler
|
|
throttledLogger log.Logger
|
|
}
|
|
|
|
// NewMetricClient creates a new instance of adminservice.AdminServiceClient that emits metrics
|
|
func NewMetricClient(
|
|
client adminservice.AdminServiceClient,
|
|
metricsHandler metrics.Handler,
|
|
throttledLogger log.Logger,
|
|
) adminservice.AdminServiceClient {
|
|
return &metricClient{
|
|
client: client,
|
|
metricsHandler: metricsHandler,
|
|
throttledLogger: throttledLogger,
|
|
}
|
|
}
|
|
|
|
func (c *metricClient) startMetricsRecording(
|
|
ctx context.Context,
|
|
operation string,
|
|
) (metrics.Handler, time.Time) {
|
|
caller := headers.GetCallerInfo(ctx).CallerName
|
|
metricsHandler := c.metricsHandler.WithTags(metrics.OperationTag(operation), metrics.NamespaceTag(caller), metrics.ServiceRoleTag(metrics.AdminRoleTagValue))
|
|
metrics.ClientRequests.With(metricsHandler).Record(1)
|
|
return metricsHandler, time.Now().UTC()
|
|
}
|
|
|
|
func (c *metricClient) finishMetricsRecording(
|
|
metricsHandler metrics.Handler,
|
|
startTime time.Time,
|
|
err error,
|
|
) {
|
|
if err != nil {
|
|
switch err.(type) {
|
|
case *serviceerror.Canceled,
|
|
*serviceerror.DeadlineExceeded,
|
|
*serviceerror.NotFound,
|
|
*serviceerror.QueryFailed,
|
|
*serviceerror.NamespaceNotFound,
|
|
*serviceerror.WorkflowNotReady,
|
|
*serviceerror.WorkflowExecutionAlreadyStarted,
|
|
*serviceerror.ResourceExhausted:
|
|
// noop - not interest and too many logs
|
|
default:
|
|
c.throttledLogger.Info("admin client encountered error", tag.Error(err), tag.ServiceErrorType(err))
|
|
}
|
|
metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err))
|
|
}
|
|
metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime))
|
|
}
|
|
|
|
func (c *metricClient) StreamWorkflowReplicationMessages(
|
|
ctx context.Context,
|
|
opts ...grpc.CallOption,
|
|
) (_ adminservice.AdminService_StreamWorkflowReplicationMessagesClient, retError error) {
|
|
|
|
metricsHandler, startTime := c.startMetricsRecording(ctx, metrics.AdminClientStreamWorkflowReplicationMessagesScope)
|
|
defer func() {
|
|
c.finishMetricsRecording(metricsHandler, startTime, retError)
|
|
}()
|
|
|
|
return c.client.StreamWorkflowReplicationMessages(ctx, opts...)
|
|
}
|