mirror of
https://github.com/hanzoai/tasks.git
synced 2026-11-02 12:22:18 +00:00
Lanie HeiandGitHub
5f5a0f6a4d
## What changed? -> Part I (History service): @dnr and others have recently implemented a health check to speed up the history service rollout by explicitly marking it as healthy when the history shards have stabilized after startup instead of waiting an arbitrary time period. In this PR, I've reused this check in the history service to ensure that the history service has marked the host as healthy. This avoids a false positive because the host isn't yet receiving error metrics, and so it might not be accurate to say it's serving if history shards haven't ceased shuffling. Added health status starting to differentiate from unhealthy host status to use this upstream and help with error propagation and debugging. (footnote) Further background on the host serving status for documentation and general interest: The status is set [here](https://github.com/temporalio/temporal/blob/main/service/history/service.go#L82) after it has both joined ringpop, and the shards have stabiliized. It reuses a gRPC history health check that was previously implemented, but was unused. -> Part II (Frontend service, aggregator of history.DeepHealthCheck): In the frontend service, I've added the default percentage of 5% based on the guidance from @00Sim that this would be quite a high error percentage, accounting for rollouts. I've added this percentage to the NOT_SERVING metric to avoid a combined outage where some pods are completely down, while some are crashlooping. ## Why? This PR starts to seek improvement for failure detection in the case of crashloops. It's a small improvement as part of the effort to detect crashloopbackoff by not marking pods as healthy until they've been marked as such by the service while starting up. If the pods do not ever "start up", then it's a good indication that something is wrong. ## How did you test it? We can test during rollout by ensuring the failure detection is working as expected. ## Potential risks If the existing check is not marking pods as healthy, we could break the autodetection code. This is not directly workflow (customer)-impacting. ## Documentation N/A ## Is hotfix candidate? No
156 lines
4.4 KiB
Go
156 lines
4.4 KiB
Go
package frontend
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/suite"
|
|
enumsspb "go.temporal.io/server/api/enums/v1"
|
|
"go.temporal.io/server/common/log"
|
|
"go.temporal.io/server/common/membership"
|
|
"go.temporal.io/server/common/primitives"
|
|
"go.uber.org/mock/gomock"
|
|
)
|
|
|
|
type (
|
|
healthCheckerSuite struct {
|
|
suite.Suite
|
|
controller *gomock.Controller
|
|
|
|
membershipMonitor *membership.MockMonitor
|
|
resolver *membership.MockServiceResolver
|
|
|
|
checker *healthCheckerImpl
|
|
}
|
|
)
|
|
|
|
func TestHealthCheckerSuite(t *testing.T) {
|
|
s := new(healthCheckerSuite)
|
|
suite.Run(t, s)
|
|
}
|
|
|
|
func (s *healthCheckerSuite) SetupTest() {
|
|
s.controller = gomock.NewController(s.T())
|
|
s.membershipMonitor = membership.NewMockMonitor(s.controller)
|
|
s.resolver = membership.NewMockServiceResolver(s.controller)
|
|
s.membershipMonitor.EXPECT().GetResolver(gomock.Any()).Return(s.resolver, nil).AnyTimes()
|
|
|
|
checker := NewHealthChecker(
|
|
primitives.HistoryService,
|
|
s.membershipMonitor,
|
|
func() float64 {
|
|
return 0.25
|
|
},
|
|
func() float64 {
|
|
return 0.15
|
|
},
|
|
func(ctx context.Context, hostAddress string) (enumsspb.HealthState, error) {
|
|
switch hostAddress {
|
|
case "1", "3":
|
|
return enumsspb.HEALTH_STATE_SERVING, nil
|
|
case "2":
|
|
return enumsspb.HEALTH_STATE_UNSPECIFIED, fmt.Errorf("test")
|
|
case "4":
|
|
return enumsspb.HEALTH_STATE_DECLINED_SERVING, nil
|
|
default:
|
|
return enumsspb.HEALTH_STATE_NOT_SERVING, nil
|
|
}
|
|
},
|
|
log.NewNoopLogger(),
|
|
)
|
|
healthChecker, ok := checker.(*healthCheckerImpl)
|
|
if !ok {
|
|
s.Fail("The constructor did not return correct type")
|
|
}
|
|
s.checker = healthChecker
|
|
}
|
|
|
|
func (s *healthCheckerSuite) TearDownTest() {
|
|
s.controller.Finish()
|
|
}
|
|
|
|
func (s *healthCheckerSuite) Test_Check_Serving() {
|
|
s.resolver.EXPECT().AvailableMembers().Return([]membership.HostInfo{
|
|
membership.NewHostInfoFromAddress("1"),
|
|
membership.NewHostInfoFromAddress("2"),
|
|
membership.NewHostInfoFromAddress("3"),
|
|
membership.NewHostInfoFromAddress("1"),
|
|
})
|
|
|
|
state, err := s.checker.Check(context.Background())
|
|
s.NoError(err)
|
|
s.Equal(enumsspb.HEALTH_STATE_SERVING, state)
|
|
}
|
|
|
|
func (s *healthCheckerSuite) Test_Check_Not_Serving() {
|
|
s.resolver.EXPECT().AvailableMembers().Return([]membership.HostInfo{
|
|
membership.NewHostInfoFromAddress("1"),
|
|
membership.NewHostInfoFromAddress("2"),
|
|
membership.NewHostInfoFromAddress("3"),
|
|
membership.NewHostInfoFromAddress("4"),
|
|
membership.NewHostInfoFromAddress("5"),
|
|
})
|
|
|
|
state, err := s.checker.Check(context.Background())
|
|
s.NoError(err)
|
|
s.Equal(enumsspb.HEALTH_STATE_NOT_SERVING, state)
|
|
}
|
|
|
|
func (s *healthCheckerSuite) Test_Check_Declined_Serving() {
|
|
s.resolver.EXPECT().AvailableMembers().Return([]membership.HostInfo{
|
|
membership.NewHostInfoFromAddress("1"),
|
|
membership.NewHostInfoFromAddress("2"),
|
|
membership.NewHostInfoFromAddress("4"),
|
|
membership.NewHostInfoFromAddress("4"),
|
|
membership.NewHostInfoFromAddress("4"),
|
|
membership.NewHostInfoFromAddress("4"),
|
|
membership.NewHostInfoFromAddress("7"),
|
|
})
|
|
|
|
state, err := s.checker.Check(context.Background())
|
|
s.NoError(err)
|
|
s.Equal(enumsspb.HEALTH_STATE_DECLINED_SERVING, state)
|
|
}
|
|
|
|
func (s *healthCheckerSuite) Test_GetProportionOfNotReadyHosts() {
|
|
testCases := []struct {
|
|
name string
|
|
proportionOfDeclinedServingHosts float64
|
|
totalHosts int
|
|
expectedProportion float64
|
|
}{
|
|
{
|
|
name: "zero proportion",
|
|
proportionOfDeclinedServingHosts: 0.0,
|
|
totalHosts: 10,
|
|
expectedProportion: 0.2,
|
|
},
|
|
{
|
|
name: "small proportion with few hosts",
|
|
proportionOfDeclinedServingHosts: 0.1,
|
|
totalHosts: 10,
|
|
expectedProportion: 0.2, // 2/10 = 0.2 since numHostsToFail < 2
|
|
},
|
|
{
|
|
name: "small proportion with many hosts",
|
|
proportionOfDeclinedServingHosts: 0.1,
|
|
totalHosts: 100,
|
|
expectedProportion: 0.1, // 10 hosts > 2, so use original proportion
|
|
},
|
|
{
|
|
name: "large proportion",
|
|
proportionOfDeclinedServingHosts: 0.8,
|
|
totalHosts: 10,
|
|
expectedProportion: 0.8, // 8 hosts > 2, so use original proportion
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
s.Run(tc.name, func() {
|
|
proportion := ensureMinimumProportionOfHosts(tc.proportionOfDeclinedServingHosts, tc.totalHosts)
|
|
s.Equal(tc.expectedProportion, proportion)
|
|
})
|
|
}
|
|
}
|