mirror of
https://github.com/hanzoai/tasks.git
synced 2026-11-03 03:56:04 +00:00
Lanie HeiandGitHub
5f5a0f6a4d
## What changed? -> Part I (History service): @dnr and others have recently implemented a health check to speed up the history service rollout by explicitly marking it as healthy when the history shards have stabilized after startup instead of waiting an arbitrary time period. In this PR, I've reused this check in the history service to ensure that the history service has marked the host as healthy. This avoids a false positive because the host isn't yet receiving error metrics, and so it might not be accurate to say it's serving if history shards haven't ceased shuffling. Added health status starting to differentiate from unhealthy host status to use this upstream and help with error propagation and debugging. (footnote) Further background on the host serving status for documentation and general interest: The status is set [here](https://github.com/temporalio/temporal/blob/main/service/history/service.go#L82) after it has both joined ringpop, and the shards have stabiliized. It reuses a gRPC history health check that was previously implemented, but was unused. -> Part II (Frontend service, aggregator of history.DeepHealthCheck): In the frontend service, I've added the default percentage of 5% based on the guidance from @00Sim that this would be quite a high error percentage, accounting for rollouts. I've added this percentage to the NOT_SERVING metric to avoid a combined outage where some pods are completely down, while some are crashlooping. ## Why? This PR starts to seek improvement for failure detection in the case of crashloops. It's a small improvement as part of the effort to detect crashloopbackoff by not marking pods as healthy until they've been marked as such by the service while starting up. If the pods do not ever "start up", then it's a good indication that something is wrong. ## How did you test it? We can test during rollout by ensuring the failure detection is working as expected. ## Potential risks If the existing check is not marking pods as healthy, we could break the autodetection code. This is not directly workflow (customer)-impacting. ## Documentation N/A ## Is hotfix candidate? No
111 lines
4.0 KiB
Go
111 lines
4.0 KiB
Go
package frontend
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
|
|
enumsspb "go.temporal.io/server/api/enums/v1"
|
|
"go.temporal.io/server/common/dynamicconfig"
|
|
"go.temporal.io/server/common/log"
|
|
"go.temporal.io/server/common/log/tag"
|
|
"go.temporal.io/server/common/membership"
|
|
"go.temporal.io/server/common/primitives"
|
|
)
|
|
|
|
type (
|
|
HealthChecker interface {
|
|
Check(ctx context.Context) (enumsspb.HealthState, error)
|
|
}
|
|
|
|
healthCheckerImpl struct {
|
|
serviceName primitives.ServiceName
|
|
membershipMonitor membership.Monitor
|
|
hostFailurePercentage dynamicconfig.FloatPropertyFn
|
|
hostDeclinedServingProportion dynamicconfig.FloatPropertyFn
|
|
healthCheckFn func(ctx context.Context, hostAddress string) (enumsspb.HealthState, error)
|
|
logger log.Logger
|
|
}
|
|
)
|
|
|
|
func NewHealthChecker(
|
|
serviceName primitives.ServiceName,
|
|
membershipMonitor membership.Monitor,
|
|
hostFailurePercentage dynamicconfig.FloatPropertyFn,
|
|
hostDeclinedServingProportion dynamicconfig.FloatPropertyFn,
|
|
healthCheckFn func(ctx context.Context, hostAddress string) (enumsspb.HealthState, error),
|
|
logger log.Logger,
|
|
) HealthChecker {
|
|
return &healthCheckerImpl{
|
|
serviceName: serviceName,
|
|
membershipMonitor: membershipMonitor,
|
|
hostFailurePercentage: hostFailurePercentage,
|
|
hostDeclinedServingProportion: hostDeclinedServingProportion,
|
|
healthCheckFn: healthCheckFn,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
func (h *healthCheckerImpl) Check(ctx context.Context) (enumsspb.HealthState, error) {
|
|
resolver, err := h.membershipMonitor.GetResolver(h.serviceName)
|
|
if err != nil {
|
|
return enumsspb.HEALTH_STATE_UNSPECIFIED, err
|
|
}
|
|
|
|
hosts := resolver.AvailableMembers()
|
|
if len(hosts) == 0 {
|
|
return enumsspb.HEALTH_STATE_SERVING, nil
|
|
}
|
|
|
|
receiveCh := make(chan enumsspb.HealthState, len(hosts))
|
|
for _, host := range hosts {
|
|
go func(hostAddress string) {
|
|
resp, err := h.healthCheckFn(
|
|
ctx,
|
|
hostAddress,
|
|
)
|
|
if err != nil {
|
|
h.logger.Warn("failed to ping deep health check", tag.Error(err), tag.ServerName(string(h.serviceName)))
|
|
}
|
|
receiveCh <- resp
|
|
}(host.GetAddress())
|
|
}
|
|
|
|
var failedHostCount float64
|
|
var hostDeclinedServingCount float64
|
|
for i := 0; i < len(hosts); i++ {
|
|
healthState := <-receiveCh
|
|
switch healthState {
|
|
case enumsspb.HEALTH_STATE_NOT_SERVING, enumsspb.HEALTH_STATE_UNSPECIFIED:
|
|
failedHostCount++
|
|
case enumsspb.HEALTH_STATE_DECLINED_SERVING:
|
|
hostDeclinedServingCount++
|
|
case enumsspb.HEALTH_STATE_SERVING:
|
|
// Do nothing.
|
|
}
|
|
}
|
|
close(receiveCh)
|
|
|
|
// Make sure that at lease 2 hosts must be not ready to trigger this check.
|
|
proportionOfDeclinedServiceHosts := ensureMinimumProportionOfHosts(h.hostDeclinedServingProportion(), len(hosts))
|
|
|
|
hostDeclinedServingProportion := hostDeclinedServingCount / float64(len(hosts))
|
|
if hostDeclinedServingProportion > proportionOfDeclinedServiceHosts {
|
|
h.logger.Warn("health check exceeded host declined serving proportion threshold", tag.NewFloat64("host declined serving proportion threshold", proportionOfDeclinedServiceHosts))
|
|
return enumsspb.HEALTH_STATE_DECLINED_SERVING, nil
|
|
}
|
|
|
|
failedHostCountProportion := failedHostCount / float64(len(hosts))
|
|
if failedHostCountProportion+hostDeclinedServingProportion > h.hostFailurePercentage() {
|
|
h.logger.Warn("health check exceeded host failure percentage threshold", tag.NewFloat64("host failure percentage threshold", h.hostFailurePercentage()), tag.NewFloat64("host failure percentage", failedHostCountProportion), tag.NewFloat64("host declined serving percentage", hostDeclinedServingProportion))
|
|
return enumsspb.HEALTH_STATE_NOT_SERVING, nil
|
|
}
|
|
|
|
return enumsspb.HEALTH_STATE_SERVING, nil
|
|
}
|
|
|
|
func ensureMinimumProportionOfHosts(proportionOfDeclinedServingHosts float64, totalHosts int) float64 {
|
|
const minimumHostsFailed = 2.0 // We want to ensure that at least 2 fail before we notify the upstream.
|
|
minimumProportion := minimumHostsFailed / float64(totalHosts)
|
|
return math.Max(proportionOfDeclinedServingHosts, minimumProportion)
|
|
}
|