Files
tasks/client/history/connections_mock.go
Michael SnowdenandGitHub 631740d37d Fix stale DLQ history clients (#5089)
<!-- Describe what has changed in this PR -->
**What changed?**
This PR changes the behavior of `getAnyClientConn` to round-robin
through the list of members in the history pool, instead of looking in
the connection pool.

<!-- Tell your future self why have you made these changes -->
**Why?**
This fixes an issue where DLQ commands like merge would end up using
zombie connections to history hosts that no longer exist. Why? This was
part of a small optimization I was making. I figured that we would end
up making the client pool large if we kept making connections to
different hosts by running DLQ commands over and over when we can
instead use any valid connection. However, we never remove connections
to dead hosts from this pool, so that strategy doesn't work. Instead, we
now round-robin through the list of members in the pool.

<!-- How have you verified this change? Tested locally? Added a unit
test? Checked in staging env? -->
**How did you test it?**
I added unit tests for both the connection pool and the history client.
I also fixed a broken unit test in `TestErrGetAnyHistoryHost`. In
addition, I deployed these changes to a test environment, and I verified
that we no longer get errors, even after following this sequence:
1. Deploy the change to the frontend
2. Run a bunch of DLQ admin commands (to fill up the connection pool on
the frontend)
3. Restart the history service to get all new history hosts
4. Repeat step 2 and verify that there are no more "no route to host"
errors

<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->
**Potential risks**


<!-- Is this PR a hotfix candidate or require that a notification be
sent to the broader community? (Yes/No) -->
**Is hotfix candidate?**
2023-11-10 14:52:16 +00:00

137 lines
5.1 KiB
Go

// The MIT License
//
// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved.
//
// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
// Code generated by MockGen. DO NOT EDIT.
// Source: connections.go
// Package history is a generated GoMock package.
package history
import (
reflect "reflect"
gomock "github.com/golang/mock/gomock"
grpc "google.golang.org/grpc"
)
// MockRPCFactory is a mock of RPCFactory interface.
type MockRPCFactory struct {
ctrl *gomock.Controller
recorder *MockRPCFactoryMockRecorder
}
// MockRPCFactoryMockRecorder is the mock recorder for MockRPCFactory.
type MockRPCFactoryMockRecorder struct {
mock *MockRPCFactory
}
// NewMockRPCFactory creates a new mock instance.
func NewMockRPCFactory(ctrl *gomock.Controller) *MockRPCFactory {
mock := &MockRPCFactory{ctrl: ctrl}
mock.recorder = &MockRPCFactoryMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockRPCFactory) EXPECT() *MockRPCFactoryMockRecorder {
return m.recorder
}
// CreateInternodeGRPCConnection mocks base method.
func (m *MockRPCFactory) CreateInternodeGRPCConnection(rpcAddress string) *grpc.ClientConn {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CreateInternodeGRPCConnection", rpcAddress)
ret0, _ := ret[0].(*grpc.ClientConn)
return ret0
}
// CreateInternodeGRPCConnection indicates an expected call of CreateInternodeGRPCConnection.
func (mr *MockRPCFactoryMockRecorder) CreateInternodeGRPCConnection(rpcAddress interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateInternodeGRPCConnection", reflect.TypeOf((*MockRPCFactory)(nil).CreateInternodeGRPCConnection), rpcAddress)
}
// MockconnectionPool is a mock of connectionPool interface.
type MockconnectionPool struct {
ctrl *gomock.Controller
recorder *MockconnectionPoolMockRecorder
}
// MockconnectionPoolMockRecorder is the mock recorder for MockconnectionPool.
type MockconnectionPoolMockRecorder struct {
mock *MockconnectionPool
}
// NewMockconnectionPool creates a new mock instance.
func NewMockconnectionPool(ctrl *gomock.Controller) *MockconnectionPool {
mock := &MockconnectionPool{ctrl: ctrl}
mock.recorder = &MockconnectionPoolMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockconnectionPool) EXPECT() *MockconnectionPoolMockRecorder {
return m.recorder
}
// getAllClientConns mocks base method.
func (m *MockconnectionPool) getAllClientConns() []clientConnection {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "getAllClientConns")
ret0, _ := ret[0].([]clientConnection)
return ret0
}
// getAllClientConns indicates an expected call of getAllClientConns.
func (mr *MockconnectionPoolMockRecorder) getAllClientConns() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getAllClientConns", reflect.TypeOf((*MockconnectionPool)(nil).getAllClientConns))
}
// getOrCreateClientConn mocks base method.
func (m *MockconnectionPool) getOrCreateClientConn(addr rpcAddress) clientConnection {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "getOrCreateClientConn", addr)
ret0, _ := ret[0].(clientConnection)
return ret0
}
// getOrCreateClientConn indicates an expected call of getOrCreateClientConn.
func (mr *MockconnectionPoolMockRecorder) getOrCreateClientConn(addr interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "getOrCreateClientConn", reflect.TypeOf((*MockconnectionPool)(nil).getOrCreateClientConn), addr)
}
// resetConnectBackoff mocks base method.
func (m *MockconnectionPool) resetConnectBackoff(arg0 clientConnection) {
m.ctrl.T.Helper()
m.ctrl.Call(m, "resetConnectBackoff", arg0)
}
// resetConnectBackoff indicates an expected call of resetConnectBackoff.
func (mr *MockconnectionPoolMockRecorder) resetConnectBackoff(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "resetConnectBackoff", reflect.TypeOf((*MockconnectionPool)(nil).resetConnectBackoff), arg0)
}