hanzokv/go
1
0
Fork 0
mirror of https://github.com/hanzokv/go.git synced 2026-09-24 00:31:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(sch): MaintNotifications for ClusterClient (#3643)

Browse Source 
* [maintnotif] Cluster specific handlers (#3613)

* maint notification handlers for cluster messages

* unrelax all conns

* trigger ci on feature branches

* feat(maintnotif): lazy cluster topology reload (#3614)

* lazy cluster topology reload

* fix discrepancies between options structs

* Update osscluster_lazy_reload_test.go

Co-authored-by: Copilot <[email protected]>

* Update osscluster.go

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

* feat(e2e): mock maintnotif e2e tests (#3639)

* lazy cluster topology reload

* fix discrepancies between options structs

* Update osscluster_lazy_reload_test.go

Co-authored-by: Copilot <[email protected]>

* Update osscluster.go

Co-authored-by: Copilot <[email protected]>

* wip fault with mock proxy

* make lint happy

* fix linter issues

* faster tests with mocks

* linter once again

* add complex node test

* add ci e2e

* use correct redis container

* e2e fix

---------

Co-authored-by: Copilot <[email protected]>

* fix(retry): Add retry mechanism for NOREPLICAS error (#3647)

* fix(queuedNewConn): protect against nil context (#3649)

* fix(maintnotif): fix smigrated parser and add cluster state reload interval option (#3663)

* fix SMIGRATED parsing

* fix smigrated parser

* add ClusterStateReloadInterval to ClusterOptions

* fix tests

* set default cluster reload interval to 10s

* chore(lint): format

* feat(smigrated): new format & remember original host:port (#3697)

* lazy cluster topology reload

* fix discrepancies between options structs

* Update osscluster_lazy_reload_test.go

Co-authored-by: Copilot <[email protected]>

* Update osscluster.go

Co-authored-by: Copilot <[email protected]>

* wip fault with mock proxy

* make lint happy

* fix linter issues

* faster tests with mocks

* linter once again

* add complex node test

* add ci e2e

* use correct redis container

* e2e fix

* additional e2e tests

* fix data race

* fix random shard picker

* fix e2e tests

* fix for empty endpoint

* fix case when semaphore is full, but still need to check idle

* scenario tests

* create database from config

* wip

* feat(client): store original addrs for later use

* fix(notif): change smigrated notification

* fix(lint): fix linter

* fix(smigrated): use array

* fix(e2e): wip

* Update options.go

Co-authored-by: Copilot <[email protected]>

* Update redis.go

Co-authored-by: Copilot <[email protected]>

* fix(notif): if the conn has no original addr, trigger reload with first target

* chore(e2e): wip cluster e2e

* chore(e2e): fix nil pointer from e2e tests

* chore(e2e): fix tests and reports

* chore(e2e): proper logging in e2e

* chore(e2e): add pubsub in the tests as well

* chore(e2e): mockproxy fixes

* chore(e2e): mockproxy fixes

* chore(e2e): mockproxy fixes

* chore(e2e): mockproxy fixes v3

* stop background routines

* fix(e2e): tests

---------

Co-authored-by: Copilot <[email protected]>

* chore(docs): add example app (#3651)

* chore(lint): fmt

* chore(docs): improve docs

* chore(docs): update features.md

* chore(github): remove example

* chore(maintnotif): rename option and address pr comments

* fix(e2e): command runner should use client timeout

* chore(e2e): refactor tests

* fix(e2e): set default timeout to 90m

* fix(e2e): skip tests if proxy cannot be started

---------

Co-authored-by: Copilot <[email protected]>
Co-authored-by: ofekshenawa <[email protected]>
Co-authored-by: Elena Kolevska <[email protected]>
This commit is contained in:
Nedyalko Dyakov
2026-02-13 13:25:07 +02:00
committed by GitHub
co-authored by Copilot ofekshenawa Elena Kolevska
parent f25343d281
commit b53f2b0c9d
47 changed files with 10693 additions and 689 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+1 -1
View File
@@ -4,7 +4,7 @@ on:
push:
branches: [master, v9, 'v9.*']
pull_request:
branches: [master, v9, v9.7, v9.8, 'ndyakov/*', 'ofekshenawa/*', 'htemelski-redis/*', 'ce/*']
branches: [master, v9, v9.7, v9.8, 'ndyakov/**', 'ofekshenawa/**', 'ce/**']
permissions:
contents: read
.github/workflows/test-e2e.yml
+63
View File
@@ -0,0 +1,63 @@
name: E2E Tests
on:
push:
branches: [master, v9, 'v9.*']
pull_request:
branches: [master, v9, v9.7, v9.8, 'ndyakov/**', 'ofekshenawa/**', 'ce/**']
permissions:
contents: read
jobs:
test-e2e-mock:
name: E2E Tests (Mock Proxy)
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
go-version:
- "1.23.x"
- stable
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go ${{ matrix.go-version }}
uses: actions/setup-go@v6
with:
go-version: ${{ matrix.go-version }}
- name: Start Docker services for E2E tests
run: make docker.e2e.start
- name: Wait for services to be ready
run: |
echo "Waiting for Redis to be ready..."
timeout 30 bash -c 'until docker exec redis-standalone redis-cli ping 2>/dev/null; do sleep 1; done'
echo "Waiting for cae-resp-proxy to be ready..."
timeout 30 bash -c 'until curl -s http://localhost:18100/stats > /dev/null; do sleep 1; done'
echo "All services are ready!"
- name: Run E2E tests with mock proxy
env:
E2E_SCENARIO_TESTS: "true"
run: |
go test -v ./maintnotifications/e2e/ -timeout 30m -race
continue-on-error: false
- name: Stop Docker services
if: always()
run: make docker.e2e.stop
- name: Show Docker logs on failure
if: failure()
run: |
echo "=== Redis logs ==="
docker logs redis-standalone 2>&1 | tail -100
echo "=== cae-resp-proxy logs ==="
docker logs cae-resp-proxy 2>&1 | tail -100
echo "=== proxy-fault-injector logs ==="
docker logs proxy-fault-injector 2>&1 | tail -100
Makefile
+36 -1
View File
@@ -14,6 +14,17 @@ docker.start:
docker.stop:
docker compose --profile all down
docker.e2e.start:
@echo "Starting Redis and cae-resp-proxy for E2E tests..."
docker compose --profile e2e up -d --quiet-pull
@echo "Waiting for services to be ready..."
@sleep 3
@echo "Services ready!"
docker.e2e.stop:
@echo "Stopping E2E services..."
docker compose --profile e2e down
test:
$(MAKE) docker.start
@if [ -z "$(REDIS_VERSION)" ]; then \
@@ -66,7 +77,31 @@ bench:
export REDIS_VERSION=$(REDIS_VERSION) && \
go test ./... -test.run=NONE -test.bench=. -test.benchmem -skip Example
.PHONY: all test test.ci test.ci.skip-vectorsets bench fmt
test.e2e:
@echo "Running E2E tests with auto-start proxy..."
$(MAKE) docker.e2e.start
@echo "Running tests..."
@E2E_SCENARIO_TESTS=true go test -v ./maintnotifications/e2e/ -timeout 30m || ($(MAKE) docker.e2e.stop && exit 1)
$(MAKE) docker.e2e.stop
@echo "E2E tests completed!"
test.e2e.docker:
@echo "Running Docker-compatible E2E tests..."
$(MAKE) docker.e2e.start
@echo "Running unified injector tests..."
@E2E_SCENARIO_TESTS=true go test -v -run "TestUnifiedInjector|TestCreateTestFaultInjectorLogic|TestFaultInjectorClientCreation" ./maintnotifications/e2e/ -timeout 10m || ($(MAKE) docker.e2e.stop && exit 1)
$(MAKE) docker.e2e.stop
@echo "Docker E2E tests completed!"
test.e2e.logic:
@echo "Running E2E logic tests (no proxy required)..."
@E2E_SCENARIO_TESTS=true \
REDIS_ENDPOINTS_CONFIG_PATH=/tmp/test_endpoints_verify.json \
FAULT_INJECTION_API_URL=http://localhost:8080 \
go test -v -run "TestCreateTestFaultInjectorLogic|TestFaultInjectorClientCreation" ./maintnotifications/e2e/
@echo "Logic tests completed!"
.PHONY: all test test.ci test.ci.skip-vectorsets bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop
build:
export RE_CLUSTER=$(RE_CLUSTER) && \
adapters.go
+7
View File
@@ -61,6 +61,13 @@ func (oa *optionsAdapter) GetAddr() string {
return oa.options.Addr
}
// GetNodeAddress returns the address of the Redis node as reported by the server.
// For cluster clients, this is the endpoint from CLUSTER SLOTS before any transformation.
// For standalone clients, this defaults to Addr.
func (oa *optionsAdapter) GetNodeAddress() string {
return oa.options.NodeAddress
}
// IsTLSEnabled returns true if TLS is enabled.
func (oa *optionsAdapter) IsTLSEnabled() bool {
return oa.options.TLSConfig != nil
commands_test.go
+15 -5
View File
@@ -9124,27 +9124,37 @@ var _ = Describe("Commands", func() {
Expect(err).NotTo(HaveOccurred())
old := client.ConfigGet(ctx, key).Val()
client.ConfigSet(ctx, key, "1")
// Use a higher threshold (100ms) to avoid capturing normal operations
// that could cause flakiness due to timing variations
client.ConfigSet(ctx, key, "100")
defer client.ConfigSet(ctx, key, old[key])
result, err := client.Latency(ctx).Result()
Expect(err).NotTo(HaveOccurred())
Expect(len(result)).Should(Equal(0))
err = client.Do(ctx, "DEBUG", "SLEEP", 0.01).Err()
// Use a longer sleep (150ms) to ensure it exceeds the 100ms threshold
err = client.Do(ctx, "DEBUG", "SLEEP", 0.15).Err()
Expect(err).NotTo(HaveOccurred())
result, err = client.Latency(ctx).Result()
Expect(err).NotTo(HaveOccurred())
Expect(len(result)).Should(Equal(1))
Expect(len(result)).Should(BeNumerically(">=", 1))
// reset latency by event name
err = client.LatencyReset(ctx, result[0].Name).Err()
eventName := result[0].Name
err = client.LatencyReset(ctx, eventName).Err()
Expect(err).NotTo(HaveOccurred())
// Verify the specific event was reset (not that all events are gone)
// This avoids flakiness from other operations triggering latency events
result, err = client.Latency(ctx).Result()
Expect(err).NotTo(HaveOccurred())
Expect(len(result)).Should(Equal(0))
for _, event := range result {
if event.Name == eventName {
Fail("Event " + eventName + " should have been reset")
}
}
})
})
})
docker-compose.yml
+39 -1
View File
@@ -1,6 +1,6 @@
---
x-default-image: &default-image ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:custom-21860421418-debian-amd64}
x-default-image: &default-image ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.6.0}
services:
redis:
@@ -25,6 +25,7 @@ services:
- sentinel
- all-stack
- all
- e2e
osscluster:
image: *default-image
@@ -43,6 +44,43 @@ services:
- all-stack
- all
cae-resp-proxy:
image: redislabs/client-resp-proxy:latest
container_name: cae-resp-proxy
environment:
- TARGET_HOST=redis
- TARGET_PORT=6379
- LISTEN_PORT=17000,17001,17002,17003 # 4 proxy nodes: initially show 3, swap in 4th during SMIGRATED
- LISTEN_HOST=0.0.0.0
- API_PORT=3000
- DEFAULT_INTERCEPTORS=cluster,hitless
ports:
- "17000:17000" # Proxy node 1 (host:container)
- "17001:17001" # Proxy node 2 (host:container)
- "17002:17002" # Proxy node 3 (host:container)
- "17003:17003" # Proxy node 4 (host:container) - hidden initially, swapped in during SMIGRATED
- "18100:3000" # HTTP API port (host:container)
depends_on:
- redis
profiles:
- e2e
- all
proxy-fault-injector:
build:
context: .
dockerfile: maintnotifications/e2e/cmd/proxy-fi-server/Dockerfile
container_name: proxy-fault-injector
ports:
- "15000:5000" # Fault injector API port (host:container)
depends_on:
- cae-resp-proxy
environment:
- PROXY_API_URL=http://cae-resp-proxy:3000
profiles:
- e2e
- all
osscluster-tls:
image: *default-image
platform: linux/amd64
internal/interfaces/interfaces.go
+5
View File
@@ -40,6 +40,11 @@ type OptionsInterface interface {
// GetAddr returns the connection address.
GetAddr() string
// GetNodeAddress returns the address of the Redis node as reported by the server.
// For cluster clients, this is the endpoint from CLUSTER SLOTS before any transformation.
// For standalone clients, this defaults to Addr.
GetNodeAddress() string
// IsTLSEnabled returns true if TLS is enabled.
IsTLSEnabled() bool
internal/maintnotifications/logs/log_messages.go
+28
View File
@@ -121,6 +121,9 @@ const (
UnrelaxedTimeoutMessage = "clearing relaxed timeout"
ManagerNotInitializedMessage = "manager not initialized"
FailedToMarkForHandoffMessage = "failed to mark connection for handoff"
InvalidSeqIDInSMigratingNotificationMessage = "invalid SeqID in SMIGRATING notification"
InvalidSeqIDInSMigratedNotificationMessage = "invalid SeqID in SMIGRATED notification"
TriggeringClusterStateReloadMessage = "triggering cluster state reload"
// ========================================
// used in pool/conn
@@ -633,3 +636,28 @@ func ExtractDataFromLogMessage(logMessage string) map[string]interface{} {
// If JSON parsing fails, return empty map
return result
}
// Cluster notification functions
func InvalidSeqIDInSMigratingNotification(seqID interface{}) string {
message := fmt.Sprintf("%s: %v", InvalidSeqIDInSMigratingNotificationMessage, seqID)
return appendJSONIfDebug(message, map[string]interface{}{
"seqID": fmt.Sprintf("%v", seqID),
})
}
func InvalidSeqIDInSMigratedNotification(seqID interface{}) string {
message := fmt.Sprintf("%s: %v", InvalidSeqIDInSMigratedNotificationMessage, seqID)
return appendJSONIfDebug(message, map[string]interface{}{
"seqID": fmt.Sprintf("%v", seqID),
})
}
// TriggeringClusterStateReload logs when cluster state reload is triggered (deduplicated, once per seqID)
func TriggeringClusterStateReload(seqID int64, hostPort string, slotRanges []string) string {
message := fmt.Sprintf("%s seqID=%d host:port=%s slots=%v", TriggeringClusterStateReloadMessage, seqID, hostPort, slotRanges)
return appendJSONIfDebug(message, map[string]interface{}{
"seqID": seqID,
"hostPort": hostPort,
"slotRanges": slotRanges,
})
}
internal/pool/pool.go
+20 -4
View File
@@ -244,9 +244,9 @@ func (p *ConnPool) checkMinIdleConns() {
for p.poolSize.Load() < p.cfg.PoolSize && p.idleConnsLen.Load() < p.cfg.MinIdleConns {
// Try to acquire a semaphore token
if !p.semaphore.TryAcquire() {
// Semaphore is full, can't create more connections
p.idleCheckInProgress.Store(false)
return
// Semaphore is full, can't create more connections right now
// Break out of inner loop to check if we need to retry
break
}
p.poolSize.Add(1)
@@ -786,6 +786,15 @@ func (p *ConnPool) putConnWithoutTurn(ctx context.Context, cn *Conn) {
// putConn is the internal implementation of Put that optionally frees a turn.
func (p *ConnPool) putConn(ctx context.Context, cn *Conn, freeTurn bool) {
// Guard against nil connection
if cn == nil {
internal.Logger.Printf(ctx, "putConn called with nil connection")
if freeTurn {
p.freeTurn()
}
return
}
// Process connection using the hooks system
shouldPool := true
shouldRemove := false
@@ -836,7 +845,14 @@ func (p *ConnPool) putConn(ctx context.Context, cn *Conn, freeTurn bool) {
if !transitionedToIdle {
// Fast path failed - hook might have changed state (e.g., to UNUSABLE for handoff)
// Keep the state set by the hook and pool the connection anyway
currentState := cn.GetStateMachine().GetState()
sm := cn.GetStateMachine()
if sm == nil {
// State machine is nil - connection is in an invalid state, remove it
internal.Logger.Printf(ctx, "conn[%d] has nil state machine, removing it", cn.GetID())
p.removeConnInternal(ctx, cn, errConnNotPooled, freeTurn)
return
}
currentState := sm.GetState()
switch currentState {
case StateUnusable:
// expected state, don't log it
internal_test.go
+265
View File
@@ -77,8 +77,273 @@ var _ = Describe("newClusterState", func() {
Expect(slotAddr(state.slots[3])).To(Equal(":1234"))
})
})
Describe("NodeAddress", func() {
BeforeEach(func() {
state = createClusterState([]ClusterSlot{{
Nodes: []ClusterNode{{Addr: "127.0.0.1:7001"}},
}, {
Nodes: []ClusterNode{{Addr: "127.0.0.1:7002"}},
}, {
Nodes: []ClusterNode{{Addr: "1.2.3.4:1234"}},
}, {
Nodes: []ClusterNode{{Addr: ":1234"}},
}})
})
It("preserves node address from ClusterSlots before loopback replacement", func() {
slotNodeAddress := func(slot *clusterSlot) string {
return slot.nodes[0].Client.NodeAddress()
}
// Node addresses should be preserved even when Addr is transformed
Expect(slotNodeAddress(state.slots[0])).To(Equal("127.0.0.1:7001"))
Expect(slotNodeAddress(state.slots[1])).To(Equal("127.0.0.1:7002"))
Expect(slotNodeAddress(state.slots[2])).To(Equal("1.2.3.4:1234"))
Expect(slotNodeAddress(state.slots[3])).To(Equal(":1234"))
})
It("has different Addr and NodeAddress when loopback is replaced", func() {
// For loopback addresses, Addr should be transformed but NodeAddress preserved
slot0 := state.slots[0]
Expect(slot0.nodes[0].Client.Options().Addr).To(Equal("10.10.10.10:7001"))
Expect(slot0.nodes[0].Client.NodeAddress()).To(Equal("127.0.0.1:7001"))
// For non-loopback addresses, Addr and NodeAddress should be the same
slot2 := state.slots[2]
Expect(slot2.nodes[0].Client.Options().Addr).To(Equal("1.2.3.4:1234"))
Expect(slot2.nodes[0].Client.NodeAddress()).To(Equal("1.2.3.4:1234"))
})
})
Describe("NodeAddress with FQDN", func() {
BeforeEach(func() {
// Simulate FQDN endpoints that might be resolved/transformed
state = createClusterState([]ClusterSlot{{
Nodes: []ClusterNode{{Addr: "redis-master.example.com:6379"}},
}, {
Nodes: []ClusterNode{{Addr: "redis-replica-1.example.com:6379"}},
}, {
Nodes: []ClusterNode{{Addr: "redis-replica-2.example.com:6379"}},
}})
})
It("preserves FQDN node addresses", func() {
slotNodeAddress := func(slot *clusterSlot) string {
return slot.nodes[0].Client.NodeAddress()
}
Expect(slotNodeAddress(state.slots[0])).To(Equal("redis-master.example.com:6379"))
Expect(slotNodeAddress(state.slots[1])).To(Equal("redis-replica-1.example.com:6379"))
Expect(slotNodeAddress(state.slots[2])).To(Equal("redis-replica-2.example.com:6379"))
})
})
Describe("NodeAddress with multiple nodes per slot", func() {
BeforeEach(func() {
state = createClusterState([]ClusterSlot{{
Start: 0,
End: 5460,
Nodes: []ClusterNode{
{Addr: "127.0.0.1:7001"}, // master
{Addr: "127.0.0.1:7004"}, // replica
},
}, {
Start: 5461,
End: 10922,
Nodes: []ClusterNode{
{Addr: "master-2.redis.local:6379"}, // master
{Addr: "replica-2.redis.local:6379"}, // replica
},
}})
})
It("preserves node addresses for all nodes in a slot", func() {
// First slot - loopback addresses
slot0 := state.slots[0]
Expect(slot0.nodes[0].Client.NodeAddress()).To(Equal("127.0.0.1:7001"))
Expect(slot0.nodes[1].Client.NodeAddress()).To(Equal("127.0.0.1:7004"))
// Verify Addr is transformed for loopback
Expect(slot0.nodes[0].Client.Options().Addr).To(Equal("10.10.10.10:7001"))
Expect(slot0.nodes[1].Client.Options().Addr).To(Equal("10.10.10.10:7004"))
// Second slot - FQDN addresses (no transformation)
slot1 := state.slots[1]
Expect(slot1.nodes[0].Client.NodeAddress()).To(Equal("master-2.redis.local:6379"))
Expect(slot1.nodes[1].Client.NodeAddress()).To(Equal("replica-2.redis.local:6379"))
// Verify Addr is same as NodeAddress for non-loopback
Expect(slot1.nodes[0].Client.Options().Addr).To(Equal("master-2.redis.local:6379"))
Expect(slot1.nodes[1].Client.Options().Addr).To(Equal("replica-2.redis.local:6379"))
})
})
})
// TestNodeAddress tests that NodeAddress is correctly preserved
// when cluster nodes are created from ClusterSlots responses.
func TestNodeAddress(t *testing.T) {
t.Run("preserves node address when loopback is replaced", func(t *testing.T) {
opt := &ClusterOptions{}
opt.init()
nodes := newClusterNodes(opt)
defer nodes.Close()
// Create cluster state with loopback addresses
// Origin is non-loopback, so loopback addresses will be replaced
slots := []ClusterSlot{{
Start: 0,
End: 5460,
Nodes: []ClusterNode{{Addr: "127.0.0.1:7001"}},
}, {
Start: 5461,
End: 10922,
Nodes: []ClusterNode{{Addr: "127.0.0.1:7002"}},
}}
state, err := newClusterState(nodes, slots, "10.10.10.10:1234")
if err != nil {
t.Fatalf("newClusterState failed: %v", err)
}
// Verify Addr is transformed (loopback replaced with origin host)
if got := state.slots[0].nodes[0].Client.Options().Addr; got != "10.10.10.10:7001" {
t.Errorf("Addr = %q, want %q", got, "10.10.10.10:7001")
}
if got := state.slots[1].nodes[0].Client.Options().Addr; got != "10.10.10.10:7002" {
t.Errorf("Addr = %q, want %q", got, "10.10.10.10:7002")
}
// Verify NodeAddress is preserved (original from ClusterSlots)
if got := state.slots[0].nodes[0].Client.NodeAddress(); got != "127.0.0.1:7001" {
t.Errorf("NodeAddress = %q, want %q", got, "127.0.0.1:7001")
}
if got := state.slots[1].nodes[0].Client.NodeAddress(); got != "127.0.0.1:7002" {
t.Errorf("NodeAddress = %q, want %q", got, "127.0.0.1:7002")
}
})
t.Run("preserves FQDN node addresses", func(t *testing.T) {
opt := &ClusterOptions{}
opt.init()
nodes := newClusterNodes(opt)
defer nodes.Close()
slots := []ClusterSlot{{
Start: 0,
End: 5460,
Nodes: []ClusterNode{{Addr: "redis-master.example.com:6379"}},
}, {
Start: 5461,
End: 10922,
Nodes: []ClusterNode{{Addr: "redis-replica.example.com:6379"}},
}}
state, err := newClusterState(nodes, slots, "10.10.10.10:1234")
if err != nil {
t.Fatalf("newClusterState failed: %v", err)
}
// For non-loopback addresses, Addr and NodeAddress should be the same
if got := state.slots[0].nodes[0].Client.Options().Addr; got != "redis-master.example.com:6379" {
t.Errorf("Addr = %q, want %q", got, "redis-master.example.com:6379")
}
if got := state.slots[0].nodes[0].Client.NodeAddress(); got != "redis-master.example.com:6379" {
t.Errorf("NodeAddress = %q, want %q", got, "redis-master.example.com:6379")
}
if got := state.slots[1].nodes[0].Client.Options().Addr; got != "redis-replica.example.com:6379" {
t.Errorf("Addr = %q, want %q", got, "redis-replica.example.com:6379")
}
if got := state.slots[1].nodes[0].Client.NodeAddress(); got != "redis-replica.example.com:6379" {
t.Errorf("NodeAddress = %q, want %q", got, "redis-replica.example.com:6379")
}
})
t.Run("preserves node address for multiple nodes per slot", func(t *testing.T) {
opt := &ClusterOptions{}
opt.init()
nodes := newClusterNodes(opt)
defer nodes.Close()
slots := []ClusterSlot{{
Start: 0,
End: 5460,
Nodes: []ClusterNode{
{Addr: "127.0.0.1:7001"}, // master
{Addr: "127.0.0.1:7004"}, // replica
},
}}
state, err := newClusterState(nodes, slots, "10.10.10.10:1234")
if err != nil {
t.Fatalf("newClusterState failed: %v", err)
}
// Master node
if got := state.slots[0].nodes[0].Client.Options().Addr; got != "10.10.10.10:7001" {
t.Errorf("Master Addr = %q, want %q", got, "10.10.10.10:7001")
}
if got := state.slots[0].nodes[0].Client.NodeAddress(); got != "127.0.0.1:7001" {
t.Errorf("Master NodeAddress = %q, want %q", got, "127.0.0.1:7001")
}
// Replica node
if got := state.slots[0].nodes[1].Client.Options().Addr; got != "10.10.10.10:7004" {
t.Errorf("Replica Addr = %q, want %q", got, "10.10.10.10:7004")
}
if got := state.slots[0].nodes[1].Client.NodeAddress(); got != "127.0.0.1:7004" {
t.Errorf("Replica NodeAddress = %q, want %q", got, "127.0.0.1:7004")
}
})
t.Run("GetOrCreate without node address defaults to Addr", func(t *testing.T) {
opt := &ClusterOptions{}
opt.init()
nodes := newClusterNodes(opt)
defer nodes.Close()
// GetOrCreate without node address (e.g., from MOVED/ASK error)
node, err := nodes.GetOrCreate("10.10.10.10:7001")
if err != nil {
t.Fatalf("GetOrCreate failed: %v", err)
}
// NodeAddress should default to Addr when not provided
if got := node.Client.NodeAddress(); got != "10.10.10.10:7001" {
t.Errorf("NodeAddress = %q, want %q", got, "10.10.10.10:7001")
}
// Addr should be set correctly
if got := node.Client.Options().Addr; got != "10.10.10.10:7001" {
t.Errorf("Addr = %q, want %q", got, "10.10.10.10:7001")
}
})
t.Run("GetOrCreateWithNodeAddress sets node address", func(t *testing.T) {
opt := &ClusterOptions{}
opt.init()
nodes := newClusterNodes(opt)
defer nodes.Close()
// GetOrCreateWithNodeAddress with node address
node, err := nodes.GetOrCreateWithNodeAddress("10.10.10.10:7001", "127.0.0.1:7001")
if err != nil {
t.Fatalf("GetOrCreateWithNodeAddress failed: %v", err)
}
// NodeAddress should be set
if got := node.Client.NodeAddress(); got != "127.0.0.1:7001" {
t.Errorf("NodeAddress = %q, want %q", got, "127.0.0.1:7001")
}
// Addr should be the transformed address
if got := node.Client.Options().Addr; got != "10.10.10.10:7001" {
t.Errorf("Addr = %q, want %q", got, "10.10.10.10:7001")
}
})
}
type fixedHash string
func (h fixedHash) Get(string) string {
maintnotifications/FEATURES.md
+29 -12
View File
@@ -156,18 +156,16 @@ Capped by: min(MaxActiveConns + 1, 5 × PoolSize)
### Client Support
#### Currently Supported
- **Standalone Client** (`redis.NewClient`)
- **Standalone Client** (`redis.NewClient`) - Full support for MOVING, MIGRATING, MIGRATED, FAILING_OVER, FAILED_OVER notifications
- **Cluster Client** (`redis.NewClusterClient`) - Support for SMIGRATING and SMIGRATED notifications for hitless slot migrations
#### Planned Support
- **Cluster Client** (not yet supported)
#### Will Not Support
- **Failover Client** (no planned support)
- **Ring Client** (no planned support)
## Migration Guide
### Enabling Maintenance Notifications
### Enabling Maintenance Notifications (Standalone Client)
**Before:**
```go
@@ -188,6 +186,26 @@ client := redis.NewClient(&redis.Options{
})
```
### Enabling Hitless Upgrades (Cluster Client)
For Redis Cluster with hitless slot migration support:
```go
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{"localhost:7000", "localhost:7001", "localhost:7002"},
Protocol: 3, // RESP3 required for push notifications
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeAuto,
RelaxedTimeout: 10 * time.Second, // Extended timeout during slot migrations
},
})
```
The cluster client automatically handles:
- **SMIGRATING**: Relaxes timeouts when slots are being migrated
- **SMIGRATED**: Triggers lazy cluster state reload when migration completes
- **SeqID Deduplication**: Same notification from multiple nodes triggers only one reload
### Adding Monitoring
```go
@@ -206,13 +224,12 @@ if manager != nil {
## Known Limitations
1. **Standalone Only**: Currently only supported in standalone Redis clients
2. **RESP3 Required**: Push notifications require RESP3 protocol
3. **Server Support**: Requires Redis Enterprise or compatible Redis with maintenance notifications
4. **Single Connection Commands**: Some commands (MULTI/EXEC, WATCH) may need special handling
5. **No Failover/Ring Client Support**: Failover and Ring clients are not supported and there are no plans to add support
1. **RESP3 Required**: Push notifications require RESP3 protocol
2. **Server Support**: Requires Redis Enterprise or compatible Redis with maintenance notifications
3. **Single Connection Commands**: Some commands (MULTI/EXEC, WATCH) may need special handling
4. **No Failover/Ring Client Support**: Failover and Ring clients are not supported and there are no plans to add support
## Future Enhancements
- Cluster client support
- Enhanced metrics and observability
- Enhanced metrics and observability
- TTL-based cleanup for SeqID deduplication map
maintnotifications/README.md
+8 -2
View File
@@ -2,8 +2,14 @@
Seamless Redis connection handoffs during cluster maintenance operations without dropping connections.
## ⚠️ **Important Note**
**Maintenance notifications are currently supported only in standalone Redis clients.** Cluster clients (ClusterClient, FailoverClient, etc.) do not yet support this functionality.
## Cluster Support
**Cluster notifications are now supported for ClusterClient!**
- **SMIGRATING**: `["SMIGRATING", SeqID, slot/range, ...]` - Relaxes timeouts when slots are being migrated
- **SMIGRATED**: `["SMIGRATED", SeqID, src host:port, dst host:port, slot/range, ...]` - Reloads cluster state when slot migration completes
**Note:** Other maintenance notifications (MOVING, MIGRATING, MIGRATED, FAILING_OVER, FAILED_OVER) are supported only in standalone Redis clients. Cluster clients support SMIGRATING and SMIGRATED for cluster-specific slot migration handling.
## Quick Start
maintnotifications/e2e/README_SCENARIOS.md
+46 -6
View File
@@ -7,17 +7,36 @@ This directory contains comprehensive end-to-end test scenarios for Redis push n
## Introduction
To run those tests you would need a fault injector service, please review the client and feel free to implement your
fault injector of choice. Those tests are tailored for Redis Enterprise, but can be adapted to other Redis distributions where
a fault injector is available.
These tests support two modes:
Once you have fault injector service up and running, you can execute the tests by running the `run-e2e-tests.sh` script.
there are three environment variables that need to be set before running the tests:
### 1. Mock Proxy Mode (Default)
Uses a local Docker-based proxy ([cae-resp-proxy](https://github.com/redis-developer/cae-resp-proxy)) to simulate Redis Enterprise behavior. This mode:
- Runs entirely locally without external dependencies
- Provides fast feedback for development
- Simulates cluster topology changes
- Supports SMIGRATING and SMIGRATED notifications
To run in mock proxy mode:
```bash
make test.e2e
```
### 2. Real Fault Injector Mode
Uses a real Redis Enterprise fault injector service for comprehensive testing. This mode:
- Tests against actual Redis Enterprise clusters
- Validates real-world scenarios
- Requires external fault injector setup
To run with a real fault injector, set these environment variables:
- `REDIS_ENDPOINTS_CONFIG_PATH`: Path to Redis endpoints configuration
- `FAULT_INJECTION_API_URL`: URL of the fault injector server
- `E2E_SCENARIO_TESTS`: Set to `true` to enable scenario tests
Then run:
```bash
./scripts/run-e2e-tests.sh
```
## Test Scenarios Overview
### 1. Basic Push Notifications (`scenario_push_notifications_test.go`)
@@ -44,7 +63,28 @@ there are three environment variables that need to be set before running the tes
- Notification delivery consistency
- Handoff behavior per endpoint type
### 3. Database Management Scenario (`scenario_database_management_test.go`)
### 3. Unified Injector Scenarios (`scenario_unified_injector_test.go`)
**Mock proxy-based notification testing**
- **Purpose**: Test SMIGRATING and SMIGRATED notifications with simulated cluster topology changes
- **Features Tested**:
- SMIGRATING notifications (slot migration in progress)
- SMIGRATED notifications (slot migration completed)
- Cluster topology changes (node swap simulation)
- Complex multi-step migration scenarios
- **Configuration**: Uses local Docker proxy (cae-resp-proxy) with 4 nodes
- **Duration**: ~10 seconds
- **Key Validations**:
- Notification delivery and parsing
- Cluster state reload callbacks
- Client resilience during migrations
- Topology change handling
- **Topology Simulation**:
- Starts with 4 proxy nodes (17000-17003)
- Initially exposes 3 nodes in CLUSTER SLOTS (17000, 17001, 17002)
- On SMIGRATED, swaps node 2 for node 3 (simulates node replacement)
- Verifies client continues to function after topology change
### 4. Database Management Scenario (`scenario_database_management_test.go`)
**Dynamic database creation and deletion**
- **Purpose**: Test database lifecycle management via fault injector
- **Features Tested**: CREATE_DATABASE, DELETE_DATABASE endpoints
maintnotifications/e2e/cluster_maintnotif_test.go
+1326
View File
File diff suppressed because it is too large Load Diff
maintnotifications/e2e/cmd/proxy-fi-server/Dockerfile
+43
View File
@@ -0,0 +1,43 @@
# Build stage
FROM golang:1.21-alpine AS builder
WORKDIR /build
# Copy go mod files
COPY go.mod go.sum ./
RUN go mod download
# Copy source code
COPY . .
# Build the proxy-fi-server binary
RUN cd maintnotifications/e2e/cmd/proxy-fi-server && \
CGO_ENABLED=0 GOOS=linux go build -o /proxy-fi-server .
# Runtime stage
FROM alpine:latest
RUN apk --no-cache add ca-certificates
# Create a non-root user
RUN addgroup -g 1000 appuser && \
adduser -D -u 1000 -G appuser appuser
WORKDIR /app
# Copy the binary from builder
COPY --from=builder /proxy-fi-server .
# Change ownership of the app directory
RUN chown -R appuser:appuser /app
# Switch to non-root user
USER appuser
# Expose the fault injector API port
EXPOSE 5000
# Run the server
ENTRYPOINT ["/app/proxy-fi-server"]
CMD ["--listen", "0.0.0.0:5000", "--proxy-api-url", "http://cae-resp-proxy:3000"]
maintnotifications/e2e/cmd/proxy-fi-server/main.go
+42
View File
@@ -0,0 +1,42 @@
package main
import (
"flag"
"fmt"
"os"
"os/signal"
"syscall"
e2e "github.com/redis/go-redis/v9/maintnotifications/e2e"
)
func main() {
listenAddr := flag.String("listen", "0.0.0.0:5000", "Address to listen on for fault injector API")
proxyAPIURL := flag.String("proxy-api-url", "http://localhost:18100", "URL of the cae-resp-proxy API (updated to avoid macOS Control Center conflict)")
flag.Parse()
fmt.Printf("Starting Proxy Fault Injector Server...\n")
fmt.Printf(" Listen address: %s\n", *listenAddr)
fmt.Printf(" Proxy API URL: %s\n", *proxyAPIURL)
server := e2e.NewProxyFaultInjectorServerWithURL(*listenAddr, *proxyAPIURL)
if err := server.Start(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to start server: %v\n", err)
os.Exit(1)
}
fmt.Printf("Proxy Fault Injector Server started successfully\n")
fmt.Printf("Fault Injector API available at http://%s\n", *listenAddr)
// Wait for interrupt signal
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
<-sigChan
fmt.Println("\nShutting down...")
if err := server.Stop(); err != nil {
fmt.Fprintf(os.Stderr, "Error during shutdown: %v\n", err)
os.Exit(1)
}
fmt.Println("Server stopped")
}
maintnotifications/e2e/command_runner_test.go
+19 -11
View File
@@ -32,22 +32,27 @@ type CommandRunner struct {
// NewCommandRunner creates a new command runner
func NewCommandRunner(client redis.UniversalClient) (*CommandRunner, func()) {
stopCh := make(chan struct{})
// Use buffered channel so Stop() can always send without blocking
stopCh := make(chan struct{}, 1)
return &CommandRunner{
client: client,
stopCh: stopCh,
errors: make([]error, 0),
}, func() {
stopCh <- struct{}{}
select {
case stopCh <- struct{}{}:
default:
// Already stopped
}
}
}
func (cr *CommandRunner) Stop() {
// Non-blocking send to buffered channel
select {
case cr.stopCh <- struct{}{}:
return
case <-time.After(500 * time.Millisecond):
return
default:
// Already has a stop signal pending
}
}
@@ -79,18 +84,22 @@ func (cr *CommandRunner) FireCommandsUntilStop(ctx context.Context) {
if poolSize == 0 {
poolSize = 1
}
// simulate at least 20 connections
if poolSize < 20 {
poolSize = 20
}
wg := sync.WaitGroup{}
for i := 0; i < int(poolSize); i++ {
// run 2x pool size operations
for i := 0; i < int(poolSize)*2; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
key := fmt.Sprintf("timeout-test-key-%d-%d", counter, i)
value := fmt.Sprintf("timeout-test-value-%d-%d", counter, i)
// Use a short timeout context for individual operations
opCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
err := cr.client.Set(opCtx, key, value, time.Minute).Err()
cancel()
// Use the parent context directly - let the client's configured timeouts
// (including relaxed timeouts during failover) handle timing
err := cr.client.Set(ctx, key, value, time.Minute).Err()
cr.operationCount.Add(1)
if err != nil {
@@ -103,7 +112,6 @@ func (cr *CommandRunner) FireCommandsUntilStop(ctx context.Context) {
return
}
fmt.Printf("Error: %v\n", err)
cr.errorCount.Add(1)
// Check if it's a timeout error
maintnotifications/e2e/config_autostart_logic_test.go
+181
View File
@@ -0,0 +1,181 @@
package e2e
import (
"os"
"testing"
)
// TestCreateTestFaultInjectorLogic_WithoutEnv verifies the auto-start logic
// when REDIS_ENDPOINTS_CONFIG_PATH is not set
func TestCreateTestFaultInjectorLogic_WithoutEnv(t *testing.T) {
// Save original environment
origConfigPath := os.Getenv("REDIS_ENDPOINTS_CONFIG_PATH")
origFIURL := os.Getenv("FAULT_INJECTION_API_URL")
// Clear environment to simulate no setup
os.Unsetenv("REDIS_ENDPOINTS_CONFIG_PATH")
os.Unsetenv("FAULT_INJECTION_API_URL")
// Restore environment after test
defer func() {
if origConfigPath != "" {
os.Setenv("REDIS_ENDPOINTS_CONFIG_PATH", origConfigPath)
}
if origFIURL != "" {
os.Setenv("FAULT_INJECTION_API_URL", origFIURL)
}
}()
// Test GetEnvConfig - should fail when REDIS_ENDPOINTS_CONFIG_PATH is not set
envConfig, err := GetEnvConfig()
if err == nil {
t.Fatal("Expected GetEnvConfig() to fail when REDIS_ENDPOINTS_CONFIG_PATH is not set")
}
if envConfig != nil {
t.Fatal("Expected envConfig to be nil when GetEnvConfig() fails")
}
t.Log("✅ GetEnvConfig() correctly fails when REDIS_ENDPOINTS_CONFIG_PATH is not set")
t.Log("✅ This means CreateTestFaultInjectorWithCleanup() will auto-start the proxy")
}
// TestCreateTestFaultInjectorLogic_WithEnv verifies the logic
// when REDIS_ENDPOINTS_CONFIG_PATH is set
func TestCreateTestFaultInjectorLogic_WithEnv(t *testing.T) {
// Create a temporary config file
tmpFile := "/tmp/test_endpoints.json"
content := `{
"standalone": {
"endpoints": ["redis://localhost:6379"]
}
}`
if err := os.WriteFile(tmpFile, []byte(content), 0644); err != nil {
t.Fatalf("Failed to create temp file: %v", err)
}
defer os.Remove(tmpFile)
// Save original environment
origConfigPath := os.Getenv("REDIS_ENDPOINTS_CONFIG_PATH")
origFIURL := os.Getenv("FAULT_INJECTION_API_URL")
// Set environment
os.Setenv("REDIS_ENDPOINTS_CONFIG_PATH", tmpFile)
os.Setenv("FAULT_INJECTION_API_URL", "http://test-fi:9999")
// Restore environment after test
defer func() {
if origConfigPath != "" {
os.Setenv("REDIS_ENDPOINTS_CONFIG_PATH", origConfigPath)
} else {
os.Unsetenv("REDIS_ENDPOINTS_CONFIG_PATH")
}
if origFIURL != "" {
os.Setenv("FAULT_INJECTION_API_URL", origFIURL)
} else {
os.Unsetenv("FAULT_INJECTION_API_URL")
}
}()
// Test GetEnvConfig - should succeed when REDIS_ENDPOINTS_CONFIG_PATH is set
envConfig, err := GetEnvConfig()
if err != nil {
t.Fatalf("Expected GetEnvConfig() to succeed when REDIS_ENDPOINTS_CONFIG_PATH is set, got error: %v", err)
}
if envConfig == nil {
t.Fatal("Expected envConfig to be non-nil when GetEnvConfig() succeeds")
}
// Verify the fault injector URL is correct
if envConfig.FaultInjectorURL != "http://test-fi:9999" {
t.Errorf("Expected FaultInjectorURL to be 'http://test-fi:9999', got '%s'", envConfig.FaultInjectorURL)
}
t.Log("✅ GetEnvConfig() correctly succeeds when REDIS_ENDPOINTS_CONFIG_PATH is set")
t.Log("✅ This means CreateTestFaultInjectorWithCleanup() will use the real fault injector")
t.Logf("✅ Fault injector URL: %s", envConfig.FaultInjectorURL)
}
// TestCreateTestFaultInjectorLogic_DefaultFIURL verifies the default fault injector URL
func TestCreateTestFaultInjectorLogic_DefaultFIURL(t *testing.T) {
// Create a temporary config file
tmpFile := "/tmp/test_endpoints2.json"
content := `{
"standalone": {
"endpoints": ["redis://localhost:6379"]
}
}`
if err := os.WriteFile(tmpFile, []byte(content), 0644); err != nil {
t.Fatalf("Failed to create temp file: %v", err)
}
defer os.Remove(tmpFile)
// Save original environment
origConfigPath := os.Getenv("REDIS_ENDPOINTS_CONFIG_PATH")
origFIURL := os.Getenv("FAULT_INJECTION_API_URL")
// Set only config path, not fault injector URL
os.Setenv("REDIS_ENDPOINTS_CONFIG_PATH", tmpFile)
os.Unsetenv("FAULT_INJECTION_API_URL")
// Restore environment after test
defer func() {
if origConfigPath != "" {
os.Setenv("REDIS_ENDPOINTS_CONFIG_PATH", origConfigPath)
} else {
os.Unsetenv("REDIS_ENDPOINTS_CONFIG_PATH")
}
if origFIURL != "" {
os.Setenv("FAULT_INJECTION_API_URL", origFIURL)
}
}()
// Test GetEnvConfig - should succeed and use default FI URL
envConfig, err := GetEnvConfig()
if err != nil {
t.Fatalf("Expected GetEnvConfig() to succeed, got error: %v", err)
}
// Verify the default fault injector URL
if envConfig.FaultInjectorURL != "http://localhost:8080" {
t.Errorf("Expected default FaultInjectorURL to be 'http://localhost:8080', got '%s'", envConfig.FaultInjectorURL)
}
t.Log("✅ GetEnvConfig() uses default fault injector URL when FAULT_INJECTION_API_URL is not set")
t.Logf("✅ Default fault injector URL: %s", envConfig.FaultInjectorURL)
}
// TestFaultInjectorClientCreation verifies that FaultInjectorClient can be created
func TestFaultInjectorClientCreation(t *testing.T) {
// Test creating client with different URLs
testCases := []struct {
name string
url string
}{
{"localhost", "http://localhost:15000"}, // Updated to avoid macOS Control Center conflict
{"with port", "http://test:9999"},
{"with trailing slash", "http://test:9999/"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
client := NewFaultInjectorClient(tc.url)
if client == nil {
t.Fatal("Expected non-nil client")
}
// Verify the base URL (should have trailing slash removed)
expectedURL := tc.url
if expectedURL[len(expectedURL)-1] == '/' {
expectedURL = expectedURL[:len(expectedURL)-1]
}
if client.GetBaseURL() != expectedURL {
t.Errorf("Expected base URL '%s', got '%s'", expectedURL, client.GetBaseURL())
}
t.Logf("✅ Client created successfully with URL: %s", client.GetBaseURL())
})
}
}
maintnotifications/e2e/config_parser_test.go
+617 -101
View File
@@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"math/rand"
"net/http"
"net/url"
"os"
"strconv"
@@ -18,6 +19,17 @@ import (
"github.com/redis/go-redis/v9/maintnotifications"
)
// debugE2E returns true if E2E_DEBUG environment variable is set to "true"
// Use this to control verbose debug logging in e2e tests
func debugE2E() bool {
return os.Getenv("E2E_DEBUG") == "true"
}
// isPortUnavailableErrorString checks if the error string indicates a port unavailable error
func isPortUnavailableErrorString(errStr string) bool {
return strings.Contains(errStr, "port_unavailable") || strings.Contains(errStr, "Unavailable or invalid port")
}
// DatabaseEndpoint represents a single database endpoint configuration
type DatabaseEndpoint struct {
Addr []string `json:"addr"`
@@ -39,6 +51,7 @@ type EnvDatabaseConfig struct {
CertificatesLocation string `json:"certificatesLocation,omitempty"`
RawEndpoints []DatabaseEndpoint `json:"raw_endpoints,omitempty"`
Endpoints []string `json:"endpoints"`
OSSCluster bool `json:"oss_cluster,omitempty"`
}
// EnvDatabasesConfig represents the complete configuration file structure
@@ -60,6 +73,7 @@ type RedisConnectionConfig struct {
BdbID int
CertificatesLocation string
Endpoints []string
IsClusterMode bool // Force cluster mode even with single endpoint
}
// GetEnvConfig reads environment variables required for the test scenario
@@ -245,6 +259,7 @@ func ConvertEnvDatabaseConfigToRedisConnectionConfig(dbConfig EnvDatabaseConfig)
BdbID: bdbId,
CertificatesLocation: dbConfig.CertificatesLocation,
Endpoints: dbConfig.Endpoints,
IsClusterMode: dbConfig.OSSCluster,
}, nil
}
@@ -265,16 +280,17 @@ func NewClientFactory(config *RedisConnectionConfig) *ClientFactory {
// CreateClientOptions represents options for creating Redis clients
type CreateClientOptions struct {
Protocol int
MaintNotificationsConfig *maintnotifications.Config
MaxRetries int
PoolSize int
MinIdleConns int
MaxActiveConns int
ClientName string
DB int
ReadTimeout time.Duration
WriteTimeout time.Duration
Protocol int
MaintNotificationsConfig *maintnotifications.Config
MaxRetries int
PoolSize int
MinIdleConns int
MaxActiveConns int
ClientName string
DB int
ReadTimeout time.Duration
WriteTimeout time.Duration
ClusterStateReloadInterval time.Duration // For cluster clients, interval for automatic state reloads
}
// DefaultCreateClientOptions returns default options for creating Redis clients
@@ -325,16 +341,17 @@ func (cf *ClientFactory) Create(key string, options *CreateClientOptions) (redis
if len(cf.config.Endpoints) > 1 || cf.isClusterEndpoint() {
// Create cluster client
clusterOptions := &redis.ClusterOptions{
Addrs: cf.getAddresses(),
Username: cf.config.Username,
Password: cf.config.Password,
Protocol: options.Protocol,
MaintNotificationsConfig: options.MaintNotificationsConfig,
MaxRetries: options.MaxRetries,
PoolSize: options.PoolSize,
MinIdleConns: options.MinIdleConns,
MaxActiveConns: options.MaxActiveConns,
ClientName: options.ClientName,
Addrs: cf.getAddresses(),
Username: cf.config.Username,
Password: cf.config.Password,
Protocol: options.Protocol,
MaintNotificationsConfig: options.MaintNotificationsConfig,
MaxRetries: options.MaxRetries,
PoolSize: options.PoolSize,
MinIdleConns: options.MinIdleConns,
MaxActiveConns: options.MaxActiveConns,
ClientName: options.ClientName,
ClusterStateReloadInterval: 10 * time.Minute,
}
if options.ReadTimeout > 0 {
@@ -343,6 +360,9 @@ func (cf *ClientFactory) Create(key string, options *CreateClientOptions) (redis
if options.WriteTimeout > 0 {
clusterOptions.WriteTimeout = options.WriteTimeout
}
if options.ClusterStateReloadInterval > 0 {
clusterOptions.ClusterStateReloadInterval = options.ClusterStateReloadInterval
}
if cf.config.TLS {
clusterOptions.TLSConfig = &tls.Config{
@@ -465,6 +485,11 @@ func (cf *ClientFactory) GetConfig() *RedisConnectionConfig {
// isClusterEndpoint determines if the configuration represents a cluster
func (cf *ClientFactory) isClusterEndpoint() bool {
// Check if cluster mode is explicitly set
if cf.config.IsClusterMode {
return true
}
// Check if any endpoint contains cluster-related keywords
for _, endpoint := range cf.config.Endpoints {
if strings.Contains(strings.ToLower(endpoint), "cluster") {
@@ -548,13 +573,58 @@ func CreateTestClientFactoryWithBdbID(databaseName string, bdbID int) (*ClientFa
}
// CreateTestFaultInjector creates a fault injector client from environment configuration
// If REDIS_ENDPOINTS_CONFIG_PATH is not set, it automatically starts a proxy fault injector server
//
// Deprecated: Use CreateTestFaultInjectorWithCleanup instead for proper cleanup
func CreateTestFaultInjector() (*FaultInjectorClient, error) {
client, _, err := CreateTestFaultInjectorWithCleanup()
return client, err
}
// CreateTestFaultInjectorWithCleanup creates a fault injector client and returns a cleanup function
//
// Decision logic based on environment:
// 1. If REDIS_ENDPOINTS_CONFIG_PATH is set -> use real fault injector from FAULT_INJECTION_API_URL
// 2. If REDIS_ENDPOINTS_CONFIG_PATH is NOT set -> use Docker fault injector at http://localhost:15000
//
// Both the Docker proxy and fault injector should already be running (started via Docker Compose)
// This function does NOT start any services - it only connects to existing ones
//
// Usage:
//
// client, cleanup, err := CreateTestFaultInjectorWithCleanup()
// if err != nil { ... }
// defer cleanup()
func CreateTestFaultInjectorWithCleanup() (*FaultInjectorClient, func(), error) {
// Try to get environment config
envConfig, err := GetEnvConfig()
// If environment config fails, use Docker fault injector
// Note: GetEnvConfig() only fails if REDIS_ENDPOINTS_CONFIG_PATH is not set
if err != nil {
return nil, fmt.Errorf("failed to get environment config: %w", err)
// Use Docker fault injector at http://localhost:15000 (updated to avoid macOS Control Center conflict)
// The fault injector should already be running via docker-compose
faultInjectorURL := "http://localhost:15000"
// Check if fault injector is accessible
client := &http.Client{Timeout: 2 * time.Second}
resp, err := client.Get(faultInjectorURL + "/actions")
if err != nil {
return nil, nil, fmt.Errorf("Docker fault injector not accessible at %s (did you run 'make docker.e2e.start'?): %w", faultInjectorURL, err)
}
resp.Body.Close()
fmt.Printf("✓ Using Docker fault injector at %s\n", faultInjectorURL)
// Return client with no-op cleanup (Docker manages the fault injector lifecycle)
noopCleanup := func() {}
return NewFaultInjectorClient(faultInjectorURL), noopCleanup, nil
}
return NewFaultInjectorClient(envConfig.FaultInjectorURL), nil
// Using real fault injector - no cleanup needed
fmt.Printf("✓ Using real fault injector at %s\n", envConfig.FaultInjectorURL)
noopCleanup := func() {}
return NewFaultInjectorClient(envConfig.FaultInjectorURL), noopCleanup, nil
}
// GetAvailableDatabases returns a list of available database names from the configuration
@@ -573,7 +643,19 @@ func GetAvailableDatabases(configPath string) ([]string, error) {
}
// ConvertEnvDatabaseConfigToFaultInjectorConfig converts EnvDatabaseConfig to fault injector DatabaseConfig
// This creates an OSS Cluster database by default for backward compatibility.
// Use ConvertEnvDatabaseConfigToFaultInjectorConfigWithMode for explicit control over cluster mode.
func ConvertEnvDatabaseConfigToFaultInjectorConfig(envConfig EnvDatabaseConfig, name string) (DatabaseConfig, error) {
return ConvertEnvDatabaseConfigToFaultInjectorConfigWithMode(envConfig, name, true)
}
// ConvertEnvDatabaseConfigToFaultInjectorConfigWithMode converts EnvDatabaseConfig to fault injector DatabaseConfig
// with explicit control over whether to create a cluster or standalone database.
// Parameters:
// - envConfig: The environment database configuration
// - name: The name for the new database
// - clusterMode: If true, creates an OSS Cluster database (ClusterClient). If false, creates a standalone database (Client).
func ConvertEnvDatabaseConfigToFaultInjectorConfigWithMode(envConfig EnvDatabaseConfig, name string, clusterMode bool) (DatabaseConfig, error) {
var port int
// Extract port and DNS name from raw_endpoints or endpoints
@@ -598,50 +680,77 @@ func ConvertEnvDatabaseConfigToFaultInjectorConfig(envConfig EnvDatabaseConfig,
return DatabaseConfig{}, fmt.Errorf("no endpoints found in configuration")
}
randomPortOffset := 1 + rand.Intn(10) // Random port offset to avoid conflicts
randomPortOffset := 1 + rand.Intn(5) + rand.Intn(10) + rand.Intn(10) + rand.Intn(10) // Random port offset to avoid conflicts
// Build the database config for fault injector
// TODO: Make this configurable
// IT is the defaults for a sharded database at the moment
dbConfig := DatabaseConfig{
Name: name,
Port: port + randomPortOffset,
MemorySize: 268435456, // 256MB default
Replication: true,
EvictionPolicy: "noeviction",
ProxyPolicy: "single",
AutoUpgrade: true,
Sharding: true,
ShardsCount: 2,
ShardKeyRegex: []ShardKeyRegexPattern{
{Regex: ".*\\{(?<tag>.*)\\}.*"},
{Regex: "(?<tag>.*)"},
},
ShardsPlacement: "dense",
ModuleList: []DatabaseModule{
{ModuleArgs: "", ModuleName: "ReJSON"},
{ModuleArgs: "", ModuleName: "search"},
{ModuleArgs: "", ModuleName: "timeseries"},
{ModuleArgs: "", ModuleName: "bf"},
},
OSSCluster: false,
}
var dbConfig DatabaseConfig
// If we have raw_endpoints with cluster info, configure for cluster
if len(envConfig.RawEndpoints) > 0 {
endpoint := envConfig.RawEndpoints[0]
// Check if this is a cluster configuration
if endpoint.ProxyPolicy != "" && endpoint.ProxyPolicy != "single" {
dbConfig.OSSCluster = true
dbConfig.Sharding = true
dbConfig.ShardsCount = 3 // default for cluster
dbConfig.ProxyPolicy = endpoint.ProxyPolicy
dbConfig.Replication = true
if clusterMode {
// Build OSS Cluster database config for ClusterClient support
// Supports cluster-specific features like SMIGRATING notifications
dbConfig = DatabaseConfig{
Name: name,
Port: port + randomPortOffset,
MemorySize: 268435456, // 256MB default
Replication: true,
EvictionPolicy: "noeviction",
ProxyPolicy: "all-master-shards", // OSS Cluster API requires all-master-shards
AutoUpgrade: true,
Sharding: true,
ShardsCount: 2,
ShardKeyRegex: []ShardKeyRegexPattern{
{Regex: ".*\\{(?<tag>.*)\\}.*"},
{Regex: "(?<tag>.*)"},
},
ShardsPlacement: "sparse", // Use sparse placement to distribute shards across nodes (required for slot-shuffle)
ModuleList: []DatabaseModule{
{ModuleArgs: "", ModuleName: "ReJSON"},
{ModuleArgs: "", ModuleName: "search"},
{ModuleArgs: "", ModuleName: "timeseries"},
{ModuleArgs: "", ModuleName: "bf"},
},
OSSCluster: true, // Enable OSS Cluster API for ClusterClient support
}
if endpoint.OSSClusterAPIPreferredIPType != "" {
dbConfig.OSSClusterAPIPreferredIPType = endpoint.OSSClusterAPIPreferredIPType
// If we have raw_endpoints with cluster info, configure for cluster
if len(envConfig.RawEndpoints) > 0 {
endpoint := envConfig.RawEndpoints[0]
// Check if this is a cluster configuration
if endpoint.ProxyPolicy != "" && endpoint.ProxyPolicy != "single" {
dbConfig.OSSCluster = true
dbConfig.Sharding = true
dbConfig.ShardsCount = 3 // default for cluster
dbConfig.ProxyPolicy = endpoint.ProxyPolicy
dbConfig.Replication = true
}
// Note: We ignore endpoint.OSSClusterAPIPreferredIPType here because
// e2e tests run from outside the cluster network and need external IPs
}
// For OSS Cluster databases, always use external IPs since e2e tests
// run from outside the cluster network and cannot reach internal IPs
dbConfig.OSSClusterAPIPreferredIPType = "external"
} else {
// Build standalone database config for regular Client support
// Supports MOVING, MIGRATING, MIGRATED, FAILING_OVER, FAILED_OVER notifications
dbConfig = DatabaseConfig{
Name: name,
Port: port + randomPortOffset,
MemorySize: 268435456, // 256MB default
Replication: true, // Enable replication for failover support
EvictionPolicy: "noeviction",
ProxyPolicy: "single", // Single proxy for standalone database
AutoUpgrade: true,
Sharding: false, // No sharding for standalone
ShardsCount: 1, // Single shard
ModuleList: []DatabaseModule{
{ModuleArgs: "", ModuleName: "ReJSON"},
{ModuleArgs: "", ModuleName: "search"},
{ModuleArgs: "", ModuleName: "timeseries"},
{ModuleArgs: "", ModuleName: "bf"},
},
OSSCluster: false, // Disable OSS Cluster API for standalone Client support
}
}
@@ -680,12 +789,17 @@ func (m *TestDatabaseManager) CreateDatabaseFromEnvConfig(ctx context.Context, e
// CreateDatabase creates a database and waits for it to be ready
// Returns the bdb_id of the created database
// If the port is unavailable, it will retry with incremented port numbers
func (m *TestDatabaseManager) CreateDatabase(ctx context.Context, dbConfig DatabaseConfig) (int, error) {
resp, err := m.faultInjector.CreateDatabase(ctx, m.clusterIndex, dbConfig)
resp, finalPort, err := m.faultInjector.CreateDatabaseConfigWithPortRetry(ctx, m.clusterIndex, dbConfig, 100)
if err != nil {
return 0, fmt.Errorf("failed to trigger database creation: %w", err)
}
if finalPort != dbConfig.Port && debugE2E() {
fmt.Printf("[TestDatabaseManager] Database created on port %d (originally requested %d)\n", finalPort, dbConfig.Port)
}
// Wait for creation to complete
status, err := m.faultInjector.WaitForAction(ctx, resp.ActionID,
WithMaxWaitTime(5*time.Minute),
@@ -721,24 +835,69 @@ func (m *TestDatabaseManager) CreateDatabase(ctx context.Context, dbConfig Datab
// CreateDatabaseAndGetConfig creates a database and returns both the bdb_id and the full connection config from the fault injector response
// This includes endpoints, username, password, TLS settings, and raw_endpoints
// If the port is unavailable, it will retry with incremented port numbers
func (m *TestDatabaseManager) CreateDatabaseAndGetConfig(ctx context.Context, dbConfig DatabaseConfig) (int, EnvDatabaseConfig, error) {
resp, err := m.faultInjector.CreateDatabase(ctx, m.clusterIndex, dbConfig)
if err != nil {
return 0, EnvDatabaseConfig{}, fmt.Errorf("failed to trigger database creation: %w", err)
}
const maxPortRetries = 100
currentPort := dbConfig.Port
// Wait for creation to complete
status, err := m.faultInjector.WaitForAction(ctx, resp.ActionID,
WithMaxWaitTime(5*time.Minute),
WithPollInterval(5*time.Second))
if err != nil {
return 0, EnvDatabaseConfig{}, fmt.Errorf("failed to wait for database creation: %w", err)
}
var status *ActionStatusResponse
if status.Status != StatusSuccess {
// Retry loop for port unavailable errors
for attempt := 0; attempt < maxPortRetries; attempt++ {
// Update the port in the config
dbConfig.Port = currentPort
resp, err := m.faultInjector.CreateDatabase(ctx, m.clusterIndex, dbConfig)
if err != nil {
// Check if it's a port unavailable error at trigger time
if isPortUnavailableErrorString(err.Error()) {
if debugE2E() {
fmt.Printf("[TestDatabaseManager] Port %d unavailable at trigger, trying port %d\n", currentPort, currentPort+1)
}
currentPort++
continue
}
return 0, EnvDatabaseConfig{}, fmt.Errorf("failed to trigger database creation: %w", err)
}
// Wait for creation to complete
status, err = m.faultInjector.WaitForAction(ctx, resp.ActionID,
WithMaxWaitTime(5*time.Minute),
WithPollInterval(5*time.Second))
if err != nil {
return 0, EnvDatabaseConfig{}, fmt.Errorf("failed to wait for database creation: %w", err)
}
if status.Status == StatusSuccess {
// Success! Break out of retry loop
if currentPort != dbConfig.Port && debugE2E() {
fmt.Printf("[TestDatabaseManager] Database created on port %d (originally requested %d)\n", currentPort, dbConfig.Port)
}
break
}
// Check if the error is port unavailable
errorStr := ""
if status.Error != nil {
errorStr = fmt.Sprintf("%v", status.Error)
}
if isPortUnavailableErrorString(errorStr) {
if debugE2E() {
fmt.Printf("[TestDatabaseManager] Port %d unavailable after action, trying port %d\n", currentPort, currentPort+1)
}
currentPort++
continue
}
// Different error, don't retry
return 0, EnvDatabaseConfig{}, fmt.Errorf("database creation failed: %v", status.Error)
}
// Check if we exhausted retries
if status == nil || status.Status != StatusSuccess {
return 0, EnvDatabaseConfig{}, fmt.Errorf("failed to create database after %d port retries (last port tried: %d)", maxPortRetries, currentPort)
}
// Extract database configuration from output
var envConfig EnvDatabaseConfig
if status.Output == nil {
@@ -906,8 +1065,8 @@ func SetupTestDatabaseFromEnv(t *testing.T, ctx context.Context, databaseName st
t.Fatalf("Database %s not found in configuration", databaseName)
}
// Create fault injector
faultInjector, err := CreateTestFaultInjector()
// Create fault injector with cleanup
faultInjector, fiCleanup, err := CreateTestFaultInjectorWithCleanup()
if err != nil {
t.Fatalf("Failed to create fault injector: %v", err)
}
@@ -919,12 +1078,14 @@ func SetupTestDatabaseFromEnv(t *testing.T, ctx context.Context, databaseName st
testDBName := fmt.Sprintf("e2e-test-%s-%d", databaseName, time.Now().Unix())
bdbID, err = dbManager.CreateDatabaseFromEnvConfig(ctx, envDbConfig, testDBName)
if err != nil {
t.Fatalf("Failed to create test database: %v", err)
fiCleanup()
t.Fatalf("[ERROR] Failed to create test database: %v", err)
}
// Return cleanup function
// Return combined cleanup function
cleanup = func() {
dbManager.Cleanup(ctx)
fiCleanup()
}
return bdbID, cleanup
@@ -936,8 +1097,8 @@ func SetupTestDatabaseFromEnv(t *testing.T, ctx context.Context, databaseName st
// bdbID, cleanup := SetupTestDatabaseWithConfig(t, ctx, dbConfig)
// defer cleanup()
func SetupTestDatabaseWithConfig(t *testing.T, ctx context.Context, dbConfig DatabaseConfig) (bdbID int, cleanup func()) {
// Create fault injector
faultInjector, err := CreateTestFaultInjector()
// Create fault injector with cleanup
faultInjector, fiCleanup, err := CreateTestFaultInjectorWithCleanup()
if err != nil {
t.Fatalf("Failed to create fault injector: %v", err)
}
@@ -948,28 +1109,62 @@ func SetupTestDatabaseWithConfig(t *testing.T, ctx context.Context, dbConfig Dat
// Create the database
bdbID, err = dbManager.CreateDatabase(ctx, dbConfig)
if err != nil {
t.Fatalf("Failed to create test database: %v", err)
fiCleanup()
t.Fatalf("[ERROR] Failed to create test database: %v", err)
}
// Return cleanup function
// Return combined cleanup function
cleanup = func() {
dbManager.Cleanup(ctx)
fiCleanup()
}
return bdbID, cleanup
}
// SetupTestDatabaseAndFactory creates a database from environment config and returns both bdbID, factory, and cleanup function
// SetupTestDatabaseAndFactory creates a database from environment config and returns both bdbID, factory, test mode config, and cleanup function
// This is the recommended way to setup tests as it ensures the client factory connects to the newly created database
//
// If REDIS_ENDPOINTS_CONFIG_PATH is not set, it will use the Docker proxy setup (127.0.0.1:17000) instead of creating a new database.
// This allows tests to work with either the real fault injector OR the Docker proxy setup.
//
// Usage:
//
// bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
// bdbID, factory, testMode, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
// defer cleanup()
func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName string) (bdbID int, factory *ClientFactory, cleanup func()) {
func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName string) (bdbID int, factory *ClientFactory, testMode *TestModeConfig, cleanup func()) {
// Get environment config
envConfig, err := GetEnvConfig()
if err != nil {
t.Fatalf("Failed to get environment config: %v", err)
// No environment config - use Docker proxy setup
t.Logf("No environment config found, using Docker proxy setup at 127.0.0.1:17000")
// Determine cluster mode based on database name
// "standalone" creates a non-cluster client for testing FAILING_OVER/FAILED_OVER notifications
isClusterMode := databaseName != "standalone"
// Create a simple Redis connection config for Docker proxy
redisConfig := &RedisConnectionConfig{
Host: "127.0.0.1", // Use 127.0.0.1 to force IPv4
Port: 17000,
Username: "",
Password: "",
TLS: false,
BdbID: 0,
IsClusterMode: isClusterMode,
}
factory = NewClientFactory(redisConfig)
// Get proxy mock test mode config
testMode = GetTestModeConfig()
// No-op cleanup since we're not creating a database
cleanup = func() {
factory.DestroyAll()
}
return 0, factory, testMode, cleanup
}
// Get database config from environment
@@ -996,14 +1191,19 @@ func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName
t.Fatalf("Database %s not found in configuration", databaseName)
}
// Convert to DatabaseConfig
dbConfig, err := ConvertEnvDatabaseConfigToFaultInjectorConfig(envDbConfig, fmt.Sprintf("e2e-test-%s-%d", databaseName, time.Now().Unix()))
// Determine cluster mode based on database name
// "standalone" creates a non-OSS-Cluster database for testing FAILING_OVER/FAILED_OVER notifications
// Other names create OSS Cluster databases for testing SMIGRATING/SMIGRATED notifications
clusterMode := databaseName != "standalone"
// Convert to DatabaseConfig with appropriate cluster mode
dbConfig, err := ConvertEnvDatabaseConfigToFaultInjectorConfigWithMode(envDbConfig, fmt.Sprintf("e2e-test-%s-%d", databaseName, time.Now().Unix()), clusterMode)
if err != nil {
t.Fatalf("Failed to convert config: %v", err)
}
// Create fault injector
faultInjector, err := CreateTestFaultInjector()
// Create fault injector with cleanup
faultInjector, fiCleanup, err := CreateTestFaultInjectorWithCleanup()
if err != nil {
t.Fatalf("Failed to create fault injector: %v", err)
}
@@ -1014,7 +1214,8 @@ func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName
// Create the database and get the actual connection config from fault injector
bdbID, newEnvConfig, err := dbManager.CreateDatabaseAndGetConfig(ctx, dbConfig)
if err != nil {
t.Fatalf("Failed to create test database: %v", err)
fiCleanup()
t.Fatalf("[ERROR] Failed to create test database: %v", err)
}
// Use certificate location from original config if not provided by fault injector
@@ -1022,35 +1223,72 @@ func SetupTestDatabaseAndFactory(t *testing.T, ctx context.Context, databaseName
newEnvConfig.CertificatesLocation = envDbConfig.CertificatesLocation
}
// Propagate OSSCluster flag since fault injector response may not include it
newEnvConfig.OSSCluster = dbConfig.OSSCluster
// Convert EnvDatabaseConfig to RedisConnectionConfig
redisConfig, err := ConvertEnvDatabaseConfigToRedisConnectionConfig(newEnvConfig)
if err != nil {
dbManager.Cleanup(ctx)
fiCleanup()
t.Fatalf("Failed to convert database config: %v", err)
}
// Create client factory with the actual config from fault injector
factory = NewClientFactory(redisConfig)
// Get real fault injector test mode config
testMode = GetTestModeConfig()
// Combined cleanup function
cleanup = func() {
factory.DestroyAll()
dbManager.Cleanup(ctx)
fiCleanup()
}
return bdbID, factory, cleanup
return bdbID, factory, testMode, cleanup
}
// SetupTestDatabaseAndFactoryWithConfig creates a database with custom config and returns both bdbID, factory, and cleanup function
// SetupTestDatabaseAndFactoryWithConfig creates a database with custom config and returns both bdbID, factory, test mode config, and cleanup function
//
// If REDIS_ENDPOINTS_CONFIG_PATH is not set, it will use the Docker proxy setup (127.0.0.1:17000) instead of creating a new database.
// This allows tests to work with either the real fault injector OR the Docker proxy setup.
//
// Usage:
//
// bdbID, factory, cleanup := SetupTestDatabaseAndFactoryWithConfig(t, ctx, "standalone", dbConfig)
// bdbID, factory, testMode, cleanup := SetupTestDatabaseAndFactoryWithConfig(t, ctx, "standalone", dbConfig)
// defer cleanup()
func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, databaseName string, dbConfig DatabaseConfig) (bdbID int, factory *ClientFactory, cleanup func()) {
func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, databaseName string, dbConfig DatabaseConfig) (bdbID int, factory *ClientFactory, testMode *TestModeConfig, cleanup func()) {
// Get environment config to use as template for connection details
envConfig, err := GetEnvConfig()
if err != nil {
t.Fatalf("Failed to get environment config: %v", err)
// No environment config - use Docker proxy setup
t.Logf("No environment config found, using Docker proxy setup at 127.0.0.1:17000")
// Create a simple Redis connection config for Docker proxy
// The proxy simulates a cluster, so we set IsClusterMode to true
redisConfig := &RedisConnectionConfig{
Host: "127.0.0.1", // Use 127.0.0.1 to force IPv4
Port: 17000,
Username: "",
Password: "",
TLS: false,
BdbID: 0,
IsClusterMode: true, // Docker proxy simulates a cluster
}
factory = NewClientFactory(redisConfig)
// Get proxy mock test mode config
testMode = GetTestModeConfig()
// No-op cleanup since we're not creating a database
cleanup = func() {
factory.DestroyAll()
}
return 0, factory, testMode, cleanup
}
// Get database config from environment
@@ -1077,8 +1315,8 @@ func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, da
t.Fatalf("Database %s not found in configuration", databaseName)
}
// Create fault injector
faultInjector, err := CreateTestFaultInjector()
// Create fault injector with cleanup
faultInjector, fiCleanup, err := CreateTestFaultInjectorWithCleanup()
if err != nil {
t.Fatalf("Failed to create fault injector: %v", err)
}
@@ -1089,7 +1327,8 @@ func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, da
// Create the database and get the actual connection config from fault injector
bdbID, newEnvConfig, err := dbManager.CreateDatabaseAndGetConfig(ctx, dbConfig)
if err != nil {
t.Fatalf("Failed to create test database: %v", err)
fiCleanup()
t.Fatalf("[ERROR] Failed to create test database: %v", err)
}
// Use certificate location from original config if not provided by fault injector
@@ -1101,17 +1340,294 @@ func SetupTestDatabaseAndFactoryWithConfig(t *testing.T, ctx context.Context, da
redisConfig, err := ConvertEnvDatabaseConfigToRedisConnectionConfig(newEnvConfig)
if err != nil {
dbManager.Cleanup(ctx)
fiCleanup()
t.Fatalf("Failed to convert database config: %v", err)
}
// Create client factory with the actual config from fault injector
factory = NewClientFactory(redisConfig)
// Get real fault injector test mode config
testMode = GetTestModeConfig()
// Combined cleanup function
cleanup = func() {
factory.DestroyAll()
dbManager.Cleanup(ctx)
fiCleanup()
}
return bdbID, factory, cleanup
return bdbID, factory, testMode, cleanup
}
// GetSlotMigrateRequirementsCount returns the number of database requirements for a given effect/variant.
// This is useful for iterating over all required databases when running tests.
// Returns 0 if the effect/variant combination is not found or if in proxy mode (which always returns 1).
func GetSlotMigrateRequirementsCount(t *testing.T, ctx context.Context, effect SlotMigrateEffect, variant SlotMigrateVariant) int {
// Get environment config
envConfig, err := GetEnvConfig()
if err != nil {
// No environment config - use Docker proxy setup (always 1 requirement)
return 1
}
// Create fault injector client using the URL from environment config
fiClient := NewFaultInjectorClient(envConfig.FaultInjectorURL)
// Query the fault injector for the required database configuration
triggersResp, err := fiClient.GetSlotMigrateTriggers(ctx, effect, 0)
if err != nil {
t.Logf("Warning: Failed to get slot-migrate triggers: %v", err)
return 1
}
// Find the trigger that matches our variant
for _, trigger := range triggersResp.Triggers {
if trigger.Name == string(variant) || (variant == SlotMigrateVariantDefault && trigger.Name == "migrate") {
return len(trigger.Requirements)
}
}
return 1
}
// SetupTestDatabaseForSlotMigrate creates a database configured for a specific slot-migrate effect
// It queries the fault injector's GET /slot-migrate endpoint to get the required database configuration
// and creates the database with those settings.
//
// Usage:
//
// bdbID, factory, testMode, fiClient, cleanup := SetupTestDatabaseForSlotMigrate(t, ctx, SlotMigrateEffectSlotShuffle, SlotMigrateVariantMigrate)
// defer cleanup()
func SetupTestDatabaseForSlotMigrate(t *testing.T, ctx context.Context, effect SlotMigrateEffect, variant SlotMigrateVariant) (bdbID int, factory *ClientFactory, testMode *TestModeConfig, fiClient *FaultInjectorClient, cleanup func()) {
return SetupTestDatabaseForSlotMigrateWithRequirementIndex(t, ctx, effect, variant, 0)
}
// SetupTestDatabaseForSlotMigrateWithRequirementIndex creates a database configured for a specific slot-migrate effect
// using the requirement at the specified index. This allows testing against all required database configurations.
//
// Usage:
//
// reqCount := GetSlotMigrateRequirementsCount(t, ctx, effect, variant)
// for i := 0; i < reqCount; i++ {
// bdbID, factory, testMode, fiClient, cleanup := SetupTestDatabaseForSlotMigrateWithRequirementIndex(t, ctx, effect, variant, i)
// defer cleanup()
// // run test...
// }
func SetupTestDatabaseForSlotMigrateWithRequirementIndex(t *testing.T, ctx context.Context, effect SlotMigrateEffect, variant SlotMigrateVariant, requirementIndex int) (bdbID int, factory *ClientFactory, testMode *TestModeConfig, fiClient *FaultInjectorClient, cleanup func()) {
// Get environment config
envConfig, err := GetEnvConfig()
if err != nil {
// No environment config - use Docker proxy setup
t.Logf("No environment config found, using Docker proxy setup at 127.0.0.1:17000")
redisConfig := &RedisConnectionConfig{
Host: "127.0.0.1",
Port: 17000,
Username: "",
Password: "",
TLS: false,
BdbID: 0,
IsClusterMode: true,
}
factory = NewClientFactory(redisConfig)
testMode = GetTestModeConfig()
// Create fault injector client for proxy mode using Docker fault injector URL
fiClient = NewFaultInjectorClient("http://localhost:15000")
// Call CreateDatabase to initialize the proxy fault injector with 2 nodes
// This is required for slot-shuffle, remove, and remove-add effects which need >= 2 nodes
resp, err := fiClient.CreateDatabase(ctx, 0, DatabaseConfig{})
if err != nil {
t.Fatalf("Failed to trigger CreateDatabase on proxy fault injector: %v", err)
}
// Wait for the action to complete
status, err := fiClient.WaitForAction(ctx, resp.ActionID,
WithMaxWaitTime(10*time.Second),
WithPollInterval(100*time.Millisecond))
if err != nil {
t.Fatalf("Failed to wait for CreateDatabase action: %v", err)
}
if status.Status != StatusSuccess {
t.Fatalf("CreateDatabase action failed: %v", status.Error)
}
cleanup = func() {
factory.DestroyAll()
}
return 0, factory, testMode, fiClient, cleanup
}
// Create fault injector client using the URL from environment config
fiClient = NewFaultInjectorClient(envConfig.FaultInjectorURL)
// Query the fault injector for the required database configuration
t.Logf("Querying fault injector for %s effect requirements...", effect)
if debugE2E() {
slotMigrateURL := fmt.Sprintf("%s/slot-migrate?effect=%s&cluster_index=0", envConfig.FaultInjectorURL, effect)
t.Logf(" URL: %s", slotMigrateURL)
// Get raw response for debugging
rawResp, err := fiClient.GetSlotMigrateTriggersRaw(ctx, effect, 0)
if err != nil {
t.Logf("Warning: Failed to get raw slot-migrate triggers: %v", err)
} else {
t.Logf(" Raw response: %s", string(rawResp))
}
}
triggersResp, err := fiClient.GetSlotMigrateTriggers(ctx, effect, 0)
if err != nil {
t.Fatalf("Failed to get slot-migrate triggers: %v", err)
}
// Log the full response (only in debug mode)
if debugE2E() {
t.Logf(" Response: Effect=%s, Triggers=%d", triggersResp.Effect, len(triggersResp.Triggers))
for i, trigger := range triggersResp.Triggers {
t.Logf(" Trigger[%d]: Name=%s, Description=%s, Requirements=%d", i, trigger.Name, trigger.Description, len(trigger.Requirements))
for j, req := range trigger.Requirements {
t.Logf(" Requirement[%d]: DBConfig=%v, Cluster=%v, Description=%s", j, req.DBConfig, req.Cluster, req.Description)
}
}
}
// Find the trigger that matches our variant
var requiredDBConfig map[string]interface{}
for _, trigger := range triggersResp.Triggers {
if trigger.Name == string(variant) || (variant == SlotMigrateVariantDefault && trigger.Name == "migrate") {
if requirementIndex < len(trigger.Requirements) {
requiredDBConfig = trigger.Requirements[requirementIndex].DBConfig
if debugE2E() {
t.Logf("Found required DB config for %s/%s (requirement %d/%d): %v",
effect, variant, requirementIndex+1, len(trigger.Requirements), requiredDBConfig)
}
} else if len(trigger.Requirements) > 0 {
// Fall back to first requirement if index is out of bounds
requiredDBConfig = trigger.Requirements[0].DBConfig
t.Logf("Warning: requirement index %d out of bounds (max %d), using first requirement",
requirementIndex, len(trigger.Requirements)-1)
}
break
}
}
// Get database config from environment
databasesConfig, err := GetDatabaseConfigFromEnv(envConfig.RedisEndpointsConfigPath)
if err != nil {
t.Fatalf("Failed to get database config: %v", err)
}
// Get the first database config
var envDbConfig EnvDatabaseConfig
for _, config := range databasesConfig {
envDbConfig = config
break
}
// Convert to DatabaseConfig
dbConfig, err := ConvertEnvDatabaseConfigToFaultInjectorConfig(envDbConfig, fmt.Sprintf("e2e-slotmigrate-%d", time.Now().Unix()))
if err != nil {
t.Fatalf("Failed to convert config: %v", err)
}
// Apply the required configuration from fault injector
if requiredDBConfig != nil {
if shardsCount, ok := requiredDBConfig["shards_count"]; ok {
if sc, ok := shardsCount.(float64); ok {
dbConfig.ShardsCount = int(sc)
if debugE2E() {
t.Logf("Setting ShardsCount to %d (from fault injector)", dbConfig.ShardsCount)
}
}
}
if shardsPlacement, ok := requiredDBConfig["shards_placement"]; ok {
if sp, ok := shardsPlacement.(string); ok {
dbConfig.ShardsPlacement = sp
if debugE2E() {
t.Logf("Setting ShardsPlacement to %s (from fault injector)", dbConfig.ShardsPlacement)
}
}
}
if replication, ok := requiredDBConfig["replication"]; ok {
if r, ok := replication.(bool); ok {
dbConfig.Replication = r
if debugE2E() {
t.Logf("Setting Replication to %v (from fault injector)", dbConfig.Replication)
}
}
}
if ossCluster, ok := requiredDBConfig["oss_cluster"]; ok {
if oc, ok := ossCluster.(bool); ok {
dbConfig.OSSCluster = oc
if debugE2E() {
t.Logf("Setting OSSCluster to %v (from fault injector)", dbConfig.OSSCluster)
}
}
}
if sharding, ok := requiredDBConfig["sharding"]; ok {
if s, ok := sharding.(bool); ok {
dbConfig.Sharding = s
if debugE2E() {
t.Logf("Setting Sharding to %v (from fault injector)", dbConfig.Sharding)
}
}
}
}
if debugE2E() {
t.Logf("Creating database with config: ShardsCount=%d, ShardsPlacement=%s, Replication=%v, OSSCluster=%v",
dbConfig.ShardsCount, dbConfig.ShardsPlacement, dbConfig.Replication, dbConfig.OSSCluster)
}
// Create fault injector with cleanup
faultInjector, fiCleanup, err := CreateTestFaultInjectorWithCleanup()
if err != nil {
t.Fatalf("Failed to create fault injector: %v", err)
}
// Create database manager
dbManager := NewTestDatabaseManager(t, faultInjector, 0)
// Create the database and get the actual connection config from fault injector
bdbID, newEnvConfig, err := dbManager.CreateDatabaseAndGetConfig(ctx, dbConfig)
if err != nil {
fiCleanup()
t.Fatalf("[ERROR] Failed to create test database: %v", err)
}
// Use certificate location from original config if not provided by fault injector
if newEnvConfig.CertificatesLocation == "" && envDbConfig.CertificatesLocation != "" {
newEnvConfig.CertificatesLocation = envDbConfig.CertificatesLocation
}
// Propagate OSSCluster flag since fault injector response may not include it
newEnvConfig.OSSCluster = dbConfig.OSSCluster
// Convert EnvDatabaseConfig to RedisConnectionConfig
redisConfig, err := ConvertEnvDatabaseConfigToRedisConnectionConfig(newEnvConfig)
if err != nil {
dbManager.Cleanup(ctx)
fiCleanup()
t.Fatalf("Failed to convert database config: %v", err)
}
// Create client factory with the actual config from fault injector
factory = NewClientFactory(redisConfig)
// Get real fault injector test mode config
testMode = GetTestModeConfig()
// Combined cleanup function
cleanup = func() {
factory.DestroyAll()
dbManager.Cleanup(ctx)
fiCleanup()
}
return bdbID, factory, testMode, fiClient, cleanup
}
maintnotifications/e2e/fault_injector_test.go → maintnotifications/e2e/fault_injector.go
+388 -15
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
"time"
@@ -15,6 +16,14 @@ import (
// ActionType represents the type of fault injection action
type ActionType string
// ActionListItem represents a single action in the list actions response
type ActionListItem struct {
JobID string `json:"job_id"`
ActionType string `json:"action_type"`
Status string `json:"status"`
SubmittedAt string `json:"submitted_at"`
}
const (
// Redis cluster actions
ActionClusterFailover ActionType = "cluster_failover"
@@ -39,7 +48,7 @@ const (
// Redis configuration actions
ActionConfigChange ActionType = "config_change"
ActionMaintenanceMode ActionType = "maintenance_mode"
ActionSlotMigration ActionType = "slot_migration"
ActionSlotMigration ActionType = "slot_migrate"
// Sequence and complex actions
ActionSequence ActionType = "sequence_of_actions"
@@ -48,8 +57,81 @@ const (
// Database management actions
ActionDeleteDatabase ActionType = "delete_database"
ActionCreateDatabase ActionType = "create_database"
ActionFailover ActionType = "failover"
ActionMigrate ActionType = "migrate"
ActionBind ActionType = "bind"
// Slot migrate action (OSS Cluster API testing)
ActionSlotMigrate ActionType = "slot_migrate"
)
// SlotMigrateEffect represents the effect type for slot migration
type SlotMigrateEffect string
const (
// SlotMigrateEffectRemoveAdd migrates all shards from source node to empty node
// Result: One endpoint removed, one endpoint added
SlotMigrateEffectRemoveAdd SlotMigrateEffect = "remove-add"
// SlotMigrateEffectRemove migrates all shards from source node to existing node
// Result: One endpoint removed
SlotMigrateEffectRemove SlotMigrateEffect = "remove"
// SlotMigrateEffectAdd migrates one shard to empty node
// Result: One endpoint added
SlotMigrateEffectAdd SlotMigrateEffect = "add"
// SlotMigrateEffectSlotShuffle migrates one shard between existing nodes
// Result: Slots move, endpoints unchanged
SlotMigrateEffectSlotShuffle SlotMigrateEffect = "slot-shuffle"
)
// SlotMigrateVariant represents the mechanism to achieve the slot migration effect
type SlotMigrateVariant string
const (
// SlotMigrateVariantDefault is an alias for migrate
SlotMigrateVariantDefault SlotMigrateVariant = "default"
// SlotMigrateVariantMigrate uses rladmin migrate to move shards
SlotMigrateVariantMigrate SlotMigrateVariant = "migrate"
// SlotMigrateVariantFailover triggers failover to swap master/replica roles
// Requires replication to be enabled
SlotMigrateVariantFailover SlotMigrateVariant = "failover"
)
// SlotMigrateRequest represents a request to trigger a slot migration
type SlotMigrateRequest struct {
Effect SlotMigrateEffect `json:"effect"`
BdbID string `json:"bdb_id"`
ClusterIndex int `json:"cluster_index,omitempty"`
Trigger SlotMigrateVariant `json:"variant,omitempty"`
SourceNode *int `json:"source_node,omitempty"`
TargetNode *int `json:"target_node,omitempty"`
}
// SlotMigrateTrigger represents a trigger configuration for slot migration
type SlotMigrateTrigger struct {
Name string `json:"name"`
Description string `json:"description"`
Requirements []SlotMigrateTriggerRequirement `json:"requirements"`
}
// SlotMigrateTriggerRequirement represents database configuration requirements
type SlotMigrateTriggerRequirement struct {
DBConfig map[string]interface{} `json:"dbconfig"`
Cluster map[string]interface{} `json:"cluster"`
Description string `json:"description"`
}
// SlotMigrateTriggersResponse represents the response from GET /slot-migrate
type SlotMigrateTriggersResponse struct {
Effect SlotMigrateEffect `json:"effect"`
Cluster map[string]interface{} `json:"cluster"`
Triggers []SlotMigrateTrigger `json:"triggers"`
}
// ActionStatus represents the status of an action
type ActionStatus string
@@ -114,11 +196,16 @@ func (c *FaultInjectorClient) GetBaseURL() string {
return c.baseURL
}
// ActionsListResponse is the wrapper response from the /action GET endpoint
type ActionsListResponse struct {
Actions []ActionListItem `json:"actions"`
}
// ListActions lists all available actions
func (c *FaultInjectorClient) ListActions(ctx context.Context) ([]ActionType, error) {
var actions []ActionType
err := c.request(ctx, "GET", "/actions", nil, &actions)
return actions, err
func (c *FaultInjectorClient) ListActions(ctx context.Context) ([]ActionListItem, error) {
var response ActionsListResponse
err := c.request(ctx, "GET", "/action", nil, &response)
return response.Actions, err
}
// TriggerAction triggers a specific action
@@ -194,19 +281,14 @@ func (c *FaultInjectorClient) TriggerClusterFailover(ctx context.Context, nodeID
})
}
// TriggerClusterReshard triggers cluster resharding
// TriggerClusterReshard is DEPRECATED - use TriggerSlotMigrateSlotShuffle instead
// The fault injector does not support a 'cluster_reshard' action type.
// Use the /slot-migrate endpoint with effect=slot-shuffle instead.
func (c *FaultInjectorClient) TriggerClusterReshard(ctx context.Context, slots []int, sourceNode, targetNode string) (*ActionResponse, error) {
return c.TriggerAction(ctx, ActionRequest{
Type: ActionClusterReshard,
Parameters: map[string]interface{}{
"slots": slots,
"source_node": sourceNode,
"target_node": targetNode,
},
})
return nil, fmt.Errorf("TriggerClusterReshard is deprecated: action type 'cluster_reshard' does not exist in fault injector. Use TriggerSlotMigrateSlotShuffle instead")
}
// TriggerSlotMigration triggers migration of specific slots
// TriggerSlotMigration triggers migration of specific slots (legacy API)
func (c *FaultInjectorClient) TriggerSlotMigration(ctx context.Context, startSlot, endSlot int, sourceNode, targetNode string) (*ActionResponse, error) {
return c.TriggerAction(ctx, ActionRequest{
Type: ActionSlotMigration,
@@ -219,6 +301,127 @@ func (c *FaultInjectorClient) TriggerSlotMigration(ctx context.Context, startSlo
})
}
// Slot Migrate Actions (OSS Cluster API Testing)
// These methods use the /slot-migrate endpoint for testing cluster topology changes
// GetSlotMigrateTriggers returns available triggers for a slot migration effect
// This is useful for discovering what database configurations are needed for each effect/variant
func (c *FaultInjectorClient) GetSlotMigrateTriggers(ctx context.Context, effect SlotMigrateEffect, clusterIndex int) (*SlotMigrateTriggersResponse, error) {
var response SlotMigrateTriggersResponse
path := fmt.Sprintf("/slot-migrate?effect=%s&cluster_index=%d", effect, clusterIndex)
fmt.Printf("[FI] GET slot-migrate: %+v\n", path)
err := c.request(ctx, "GET", path, nil, &response)
return &response, err
}
// GetSlotMigrateTriggersRaw returns the raw JSON response from GET /slot-migrate
// This is useful for debugging when the response structure doesn't match expectations
func (c *FaultInjectorClient) GetSlotMigrateTriggersRaw(ctx context.Context, effect SlotMigrateEffect, clusterIndex int) ([]byte, error) {
path := fmt.Sprintf("/slot-migrate?effect=%s&cluster_index=%d", effect, clusterIndex)
url := c.baseURL + path
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to execute request: %w", err)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
}
if resp.StatusCode >= 400 {
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(respBody))
}
return respBody, nil
}
// TriggerSlotMigrate triggers a slot migration with the specified effect and variant
// This is the new API for testing OSS Cluster API client behavior during endpoint changes
//
// Effects:
// - remove-add: One endpoint removed, one added (migrate all shards to empty node)
// - remove: One endpoint removed (migrate all shards to existing node)
// - add: One endpoint added (migrate one shard to empty node)
// - slot-shuffle: Slots moved without endpoint change (migrate one shard between existing nodes)
//
// Variants:
// - default/migrate: Use rladmin migrate to move shards
// - maintenance_mode: Put node in maintenance mode (only for remove-add, remove)
// - failover: Trigger failover to swap master/replica roles (requires replication)
func (c *FaultInjectorClient) TriggerSlotMigrate(ctx context.Context, req SlotMigrateRequest) (*ActionResponse, error) {
var response ActionResponse
// Build query parameters
path := fmt.Sprintf("/slot-migrate?effect=%s&bdb_id=%s&cluster_index=%d",
req.Effect, req.BdbID, req.ClusterIndex)
if req.Trigger != "" {
path += fmt.Sprintf("&trigger=%s", req.Trigger)
}
if req.SourceNode != nil {
path += fmt.Sprintf("&source_node=%d", *req.SourceNode)
}
if req.TargetNode != nil {
path += fmt.Sprintf("&target_node=%d", *req.TargetNode)
}
fmt.Printf("[FI] POST slot-migrate: %+v\n %+v\n", path, req)
err := c.request(ctx, "POST", path, nil, &response)
return &response, err
}
// TriggerSlotMigrateRemoveAdd triggers a remove-add slot migration
// This migrates all shards from source node to an empty node
// Result: One endpoint removed, one endpoint added
func (c *FaultInjectorClient) TriggerSlotMigrateRemoveAdd(ctx context.Context, bdbID string, trigger SlotMigrateVariant) (*ActionResponse, error) {
return c.TriggerSlotMigrate(ctx, SlotMigrateRequest{
Effect: SlotMigrateEffectRemoveAdd,
BdbID: bdbID,
Trigger: trigger,
})
}
// TriggerSlotMigrateRemove triggers a remove slot migration
// This migrates all shards from source node to an existing node
// Result: One endpoint removed
func (c *FaultInjectorClient) TriggerSlotMigrateRemove(ctx context.Context, bdbID string, trigger SlotMigrateVariant) (*ActionResponse, error) {
return c.TriggerSlotMigrate(ctx, SlotMigrateRequest{
Effect: SlotMigrateEffectRemove,
BdbID: bdbID,
Trigger: trigger,
})
}
// TriggerSlotMigrateAdd triggers an add slot migration
// This migrates one shard to an empty node
// Result: One endpoint added
func (c *FaultInjectorClient) TriggerSlotMigrateAdd(ctx context.Context, bdbID string, trigger SlotMigrateVariant) (*ActionResponse, error) {
return c.TriggerSlotMigrate(ctx, SlotMigrateRequest{
Effect: SlotMigrateEffectAdd,
BdbID: bdbID,
Trigger: trigger,
})
}
// TriggerSlotMigrateSlotShuffle triggers a slot-shuffle migration
// This migrates one shard between existing nodes
// Result: Slots move, endpoints unchanged
func (c *FaultInjectorClient) TriggerSlotMigrateSlotShuffle(ctx context.Context, bdbID string, trigger SlotMigrateVariant) (*ActionResponse, error) {
return c.TriggerSlotMigrate(ctx, SlotMigrateRequest{
Effect: SlotMigrateEffectSlotShuffle,
BdbID: bdbID,
Trigger: trigger,
})
}
// Node Management Actions
// RestartNode restarts a specific Redis node
@@ -429,6 +632,116 @@ func (c *FaultInjectorClient) CreateDatabaseFromMap(ctx context.Context, cluster
})
}
// isPortUnavailableError checks if the error is a port unavailable error
func isPortUnavailableError(err error) bool {
if err == nil {
return false
}
errStr := err.Error()
return strings.Contains(errStr, "port_unavailable") || strings.Contains(errStr, "Unavailable or invalid port")
}
// CreateDatabaseWithPortRetry creates a new database, retrying with incremented port numbers if the port is unavailable
// Parameters:
// - clusterIndex: The index of the cluster
// - databaseConfig: The database configuration as a map (must contain "port" key)
// - maxRetries: Maximum number of port increments to try (default 100 if <= 0)
//
// Returns the ActionResponse and the final port used
func (c *FaultInjectorClient) CreateDatabaseWithPortRetry(ctx context.Context, clusterIndex int, databaseConfig map[string]interface{}, maxRetries int) (*ActionResponse, int, error) {
if maxRetries <= 0 {
maxRetries = 100
}
// Get the initial port from the config
port, ok := databaseConfig["port"]
if !ok {
// No port specified, just try once
resp, err := c.CreateDatabaseFromMap(ctx, clusterIndex, databaseConfig)
return resp, 0, err
}
// Convert port to int
var currentPort int
switch p := port.(type) {
case int:
currentPort = p
case int64:
currentPort = int(p)
case float64:
currentPort = int(p)
default:
// Can't handle this port type, just try once
resp, err := c.CreateDatabaseFromMap(ctx, clusterIndex, databaseConfig)
return resp, 0, err
}
// Try creating the database, incrementing port on failure
for attempt := 0; attempt < maxRetries; attempt++ {
// Update the port in the config
databaseConfig["port"] = currentPort
resp, err := c.CreateDatabaseFromMap(ctx, clusterIndex, databaseConfig)
if err == nil {
return resp, currentPort, nil
}
// Check if it's a port unavailable error
if !isPortUnavailableError(err) {
// Different error, don't retry
return nil, currentPort, err
}
// Port unavailable, try next port
if DebugE2E() {
fmt.Printf("[FI] Port %d unavailable, trying port %d\n", currentPort, currentPort+1)
}
currentPort++
}
return nil, currentPort, fmt.Errorf("failed to create database after %d port retries (last port tried: %d)", maxRetries, currentPort-1)
}
// CreateDatabaseConfigWithPortRetry creates a new database using DatabaseConfig, retrying with incremented port numbers if the port is unavailable
// Parameters:
// - clusterIndex: The index of the cluster
// - databaseConfig: The database configuration
// - maxRetries: Maximum number of port increments to try (default 100 if <= 0)
//
// Returns the ActionResponse and the final port used
func (c *FaultInjectorClient) CreateDatabaseConfigWithPortRetry(ctx context.Context, clusterIndex int, databaseConfig DatabaseConfig, maxRetries int) (*ActionResponse, int, error) {
if maxRetries <= 0 {
maxRetries = 100
}
currentPort := databaseConfig.Port
// Try creating the database, incrementing port on failure
for attempt := 0; attempt < maxRetries; attempt++ {
// Update the port in the config
databaseConfig.Port = currentPort
resp, err := c.CreateDatabase(ctx, clusterIndex, databaseConfig)
if err == nil {
return resp, currentPort, nil
}
// Check if it's a port unavailable error
if !isPortUnavailableError(err) {
// Different error, don't retry
return nil, currentPort, err
}
// Port unavailable, try next port
if DebugE2E() {
fmt.Printf("[FI] Port %d unavailable, trying port %d\n", currentPort, currentPort+1)
}
currentPort++
}
return nil, currentPort, fmt.Errorf("failed to create database after %d port retries (last port tried: %d)", maxRetries, currentPort-1)
}
// Complex Actions
// ExecuteSequence executes a sequence of actions
@@ -642,3 +955,63 @@ func FormatSlotRange(start, end int) string {
}
return fmt.Sprintf("%d-%d", start, end)
}
// DebugE2E returns true if E2E_DEBUG environment variable is set to "true"
// Use this to control verbose debug logging in e2e tests
func DebugE2E() bool {
return os.Getenv("E2E_DEBUG") == "true"
}
// formatSMigratingNotification formats an SMIGRATING notification in RESP3 wire format
func formatSMigratingNotification(seqID int64, slots ...string) string {
// Format: ["SMIGRATING", seqID, slot1, slot2, ...]
parts := []string{
fmt.Sprintf(">%d\r\n", len(slots)+2),
"$10\r\nSMIGRATING\r\n",
fmt.Sprintf(":%d\r\n", seqID),
}
for _, slot := range slots {
parts = append(parts, fmt.Sprintf("$%d\r\n%s\r\n", len(slot), slot))
}
return strings.Join(parts, "")
}
// formatSMigratedNotification formats an SMIGRATED notification in RESP3 wire format
func formatSMigratedNotification(seqID int64, endpoints ...string) string {
// Correct Format: ["SMIGRATED", SeqID, [[host:port, slots], [host:port, slots], ...]]
// RESP3 wire format:
// >3
// +SMIGRATED
// :SeqID
// *<num_entries>
// *2
// +<host:port>
// +<slots-or-ranges>
// Each endpoint is formatted as: "host:port slot1,slot2,range1-range2"
parts := []string{">3\r\n"}
parts = append(parts, "+SMIGRATED\r\n")
parts = append(parts, fmt.Sprintf(":%d\r\n", seqID))
count := len(endpoints)
parts = append(parts, fmt.Sprintf("*%d\r\n", count))
for _, endpoint := range endpoints {
// Split endpoint into host:port and slots
// endpoint format: "host:port slot1,slot2,range1-range2"
endpointParts := strings.SplitN(endpoint, " ", 2)
if len(endpointParts) != 2 {
continue
}
hostPort := endpointParts[0]
slots := endpointParts[1]
// Each entry is an array with 2 elements
parts = append(parts, "*2\r\n")
parts = append(parts, fmt.Sprintf("+%s\r\n", hostPort))
parts = append(parts, fmt.Sprintf("+%s\r\n", slots))
}
return strings.Join(parts, "")
}
maintnotifications/e2e/logcollector_test.go
+111 -4
View File
@@ -95,19 +95,31 @@ type MatchFunc struct {
completed atomic.Bool
F func(lstring string) bool
matches []string
matchesMu sync.Mutex // protects matches slice
found chan struct{} // channel to notify when match is found, will be closed
done func()
}
func (tlc *TestLogCollector) Printf(_ context.Context, format string, v ...interface{}) {
tlc.mu.Lock()
defer tlc.mu.Unlock()
lstr := fmt.Sprintf(format, v...)
if len(tlc.matchFuncs) > 0 {
// Check if there are match functions to process
// Use matchFuncsMutex to safely read matchFuncs
tlc.matchFuncsMutex.Lock()
hasMatchFuncs := len(tlc.matchFuncs) > 0
// Create a copy of matchFuncs to avoid holding the lock while processing
matchFuncsCopy := make([]*MatchFunc, len(tlc.matchFuncs))
copy(matchFuncsCopy, tlc.matchFuncs)
tlc.matchFuncsMutex.Unlock()
if hasMatchFuncs {
go func(lstr string) {
for _, matchFunc := range tlc.matchFuncs {
for _, matchFunc := range matchFuncsCopy {
if matchFunc.F(lstr) {
matchFunc.matchesMu.Lock()
matchFunc.matches = append(matchFunc.matches, lstr)
matchFunc.matchesMu.Unlock()
matchFunc.done()
return
}
@@ -118,6 +130,7 @@ func (tlc *TestLogCollector) Printf(_ context.Context, format string, v ...inter
fmt.Println(lstr)
}
tlc.l = append(tlc.l, fmt.Sprintf(format, v...))
tlc.mu.Unlock()
}
func (tlc *TestLogCollector) WaitForLogContaining(searchString string, timeout time.Duration) bool {
@@ -136,13 +149,25 @@ func (tlc *TestLogCollector) WaitForLogContaining(searchString string, timeout t
}
func (tlc *TestLogCollector) MatchOrWaitForLogMatchFunc(mf func(string) bool, timeout time.Duration) (string, bool) {
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] MatchOrWaitForLogMatchFunc: checking existing logs...\n")
}
if logs := tlc.GetLogsFiltered(mf); len(logs) > 0 {
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] MatchOrWaitForLogMatchFunc: found matching log in existing logs\n")
}
return logs[0], true
}
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] MatchOrWaitForLogMatchFunc: no match in existing logs, waiting with timeout %v...\n", timeout)
}
return tlc.WaitForLogMatchFunc(mf, timeout)
}
func (tlc *TestLogCollector) WaitForLogMatchFunc(mf func(string) bool, timeout time.Duration) (string, bool) {
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] WaitForLogMatchFunc: starting with timeout %v\n", timeout)
}
matchFunc := &MatchFunc{
completed: atomic.Bool{},
F: mf,
@@ -150,28 +175,74 @@ func (tlc *TestLogCollector) WaitForLogMatchFunc(mf func(string) bool, timeout t
matches: make([]string, 0),
}
matchFunc.done = func() {
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] done(): starting\n")
}
if !matchFunc.completed.CompareAndSwap(false, true) {
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] done(): already completed, returning\n")
}
return
}
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] done(): closing found channel\n")
}
close(matchFunc.found)
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] done(): acquiring matchFuncsMutex...\n")
}
tlc.matchFuncsMutex.Lock()
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] done(): matchFuncsMutex acquired\n")
}
defer tlc.matchFuncsMutex.Unlock()
for i, mf := range tlc.matchFuncs {
if mf == matchFunc {
tlc.matchFuncs = append(tlc.matchFuncs[:i], tlc.matchFuncs[i+1:]...)
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] done(): removed matchFunc from list\n")
}
return
}
}
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] done(): matchFunc not found in list\n")
}
}
tlc.matchFuncsMutex.Lock()
tlc.matchFuncs = append(tlc.matchFuncs, matchFunc)
tlc.matchFuncsMutex.Unlock()
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] WaitForLogMatchFunc: registered match func, waiting...\n")
}
select {
case <-matchFunc.found:
return matchFunc.matches[0], true
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] WaitForLogMatchFunc: match found! Acquiring matchesMu...\n")
}
matchFunc.matchesMu.Lock()
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] WaitForLogMatchFunc: matchesMu acquired, matches len=%d\n", len(matchFunc.matches))
}
defer matchFunc.matchesMu.Unlock()
if len(matchFunc.matches) > 0 {
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] WaitForLogMatchFunc: returning match\n")
}
return matchFunc.matches[0], true
}
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] WaitForLogMatchFunc: match found but no log message, this should not happen\n")
}
return "", false
case <-time.After(timeout):
if DebugE2E() {
fmt.Printf("[LOG-COLLECTOR] WaitForLogMatchFunc: TIMEOUT after %v\n", timeout)
}
// Clean up the matchFunc from the list on timeout
matchFunc.done()
return "", false
}
}
@@ -281,6 +352,8 @@ type LogAnalisis struct {
MovingCount int64
MigratingCount int64
MigratedCount int64
SMigratingCount int64
SMigratedCount int64
FailingOverCount int64
FailedOverCount int64
UnexpectedCount int64
@@ -290,6 +363,9 @@ type LogAnalisis struct {
SucceededHandoffCount int64
TotalHandoffRetries int64
TotalHandoffToCurrentEndpoint int64
// Cluster state reload tracking
ClusterStateReloadCount int64
}
func NewLogAnalysis(logs []string) *LogAnalisis {
@@ -333,6 +409,11 @@ func (la *LogAnalisis) Analyze() {
la.TotalNotifications++
switch {
// ordering here is important, since SMIGRATED contains MIGRATED and SMIGRATING contains MIGRATING
case notificationType(log, "SMIGRATING"):
la.SMigratingCount++
case notificationType(log, "SMIGRATED"):
la.SMigratedCount++
case notificationType(log, "MOVING"):
la.MovingCount++
case notificationType(log, "MIGRATING"):
@@ -349,6 +430,12 @@ func (la *LogAnalisis) Analyze() {
}
}
// Track cluster state reloads (deduplicated, once per seqID)
// Log format: "triggering cluster state reload seqID=%d host:port=%s slots=%v"
if strings.Contains(log, logs2.TriggeringClusterStateReloadMessage) {
la.ClusterStateReloadCount++
}
if strings.Contains(log, "conn[") {
connID := extractConnID(log)
if _, ok := la.connIds[connID]; !ok {
@@ -377,6 +464,14 @@ func (la *LogAnalisis) Analyze() {
}
}
// PrintSummary prints a compact summary without detailed information
func (la *LogAnalisis) PrintSummary(t *testing.T) {
t.Logf("Log Summary: %d logs, %d connections | Handoffs: %d (ok:%d fail:%d) | Notifications: %d | TimeoutErrors: %d",
len(la.logs), len(la.connIds),
la.TotalHandoffCount, la.SucceededHandoffCount, la.FailedHandoffCount,
la.TotalNotifications, la.TimeoutErrorsCount)
}
func (la *LogAnalisis) Print(t *testing.T) {
t.Logf("Log Analysis results for %d logs and %d connections:", len(la.logs), len(la.connIds))
t.Logf("Connection Count: %d", la.ConnectionCount)
@@ -407,6 +502,12 @@ func (la *LogAnalisis) Print(t *testing.T) {
t.Logf(" - FAILED_OVER: %d", la.FailedOverCount)
t.Logf(" - Unexpected: %d", la.UnexpectedCount)
t.Logf("-------------")
t.Logf("-Cluster-Specific Notification Analysis-")
t.Logf("-------------")
t.Logf(" - SMIGRATING: %d", la.SMigratingCount)
t.Logf(" - SMIGRATED: %d", la.SMigratedCount)
t.Logf(" - Cluster state reloads: %d", la.ClusterStateReloadCount)
t.Logf("-------------")
t.Logf("Log Analysis completed successfully")
}
@@ -426,6 +527,12 @@ func extractConnID(log string) uint64 {
func notificationType(log string, nt string) bool {
return strings.Contains(log, nt)
}
// notificationTypeWithSeqID matches logs that contain the notification type AND have seqID
// This matches "processing notification started" logs which include seqID in the JSON
func notificationTypeWithSeqID(log string, nt string) bool {
return strings.Contains(log, nt) && strings.Contains(log, "seqID")
}
func connID(log string, connID uint64) bool {
return strings.Contains(log, fmt.Sprintf("conn[%d]", connID))
}
maintnotifications/e2e/main_test.go
+16 -3
View File
@@ -17,6 +17,7 @@ const defaultTestTimeout = 30 * time.Minute
// Global fault injector client
var faultInjector *FaultInjectorClient
var faultInjectorCleanup func()
func TestMain(m *testing.M) {
var err error
@@ -25,10 +26,22 @@ func TestMain(m *testing.M) {
return
}
faultInjector, err = CreateTestFaultInjector()
if err != nil {
panic("Failed to create fault injector: " + err.Error())
// Only create fault injector if we're not using the unified injector
// The unified injector tests use NewNotificationInjector() which auto-detects the mode
// and doesn't require the global faultInjector variable
// We can detect unified injector mode by checking if REDIS_ENDPOINTS_CONFIG_PATH is NOT set
// (which means we're using the proxy mock mode)
if os.Getenv("REDIS_ENDPOINTS_CONFIG_PATH") != "" {
// Real fault injector mode - create the global fault injector
faultInjector, faultInjectorCleanup, err = CreateTestFaultInjectorWithCleanup()
if err != nil {
panic("Failed to create fault injector: " + err.Error())
}
defer faultInjectorCleanup()
} else {
log.Println("Using proxy mock mode - skipping global fault injector setup")
}
// use log collector to capture logs from redis clients
logCollector = NewTestLogCollector()
redis.SetLogger(logCollector)
maintnotifications/e2e/notification_injector_test.go
+676
View File
@@ -0,0 +1,676 @@
package e2e
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"time"
)
// NotificationInjector is an interface that can inject maintenance notifications
// It can be implemented by either a real fault injector or a proxy-based mock
type NotificationInjector interface {
// InjectSMIGRATING injects an SMIGRATING notification
InjectSMIGRATING(ctx context.Context, seqID int64, slots ...string) error
// InjectSMIGRATED injects an SMIGRATED notification
InjectSMIGRATED(ctx context.Context, seqID int64, hostPort string, slots ...string) error
// InjectMOVING injects a MOVING notification (for standalone)
// Format: ["MOVING", seqID, timeS, endpoint]
InjectMOVING(ctx context.Context, seqID int64, timeS int64, endpoint string) error
// InjectMIGRATING injects a MIGRATING notification (for standalone)
InjectMIGRATING(ctx context.Context, seqID int64, slot int) error
// InjectMIGRATED injects a MIGRATED notification (for standalone)
InjectMIGRATED(ctx context.Context, seqID int64, slot int) error
// InjectFAILING_OVER injects a FAILING_OVER notification
InjectFAILING_OVER(ctx context.Context, seqID int64) error
// InjectFAILED_OVER injects a FAILED_OVER notification
InjectFAILED_OVER(ctx context.Context, seqID int64) error
// Start starts the injector (if needed)
Start() error
// Stop stops the injector (if needed)
Stop() error
// GetClusterAddrs returns the cluster addresses to connect to
GetClusterAddrs() []string
// IsReal returns true if this is a real fault injector (not a mock)
IsReal() bool
// GetTestModeConfig returns the test mode configuration for this injector
GetTestModeConfig() *TestModeConfig
}
// NewNotificationInjector creates a notification injector based on environment
// If FAULT_INJECTOR_URL is set, it uses the real fault injector
// Otherwise, it uses the proxy-based mock
func NewNotificationInjector() (NotificationInjector, error) {
if faultInjectorURL := os.Getenv("FAULT_INJECTOR_URL"); faultInjectorURL != "" {
// Use real fault injector
return NewFaultInjectorNotificationInjector(faultInjectorURL), nil
}
// Use proxy-based mock
apiPort := 18100 // Default port (updated to avoid macOS Control Center conflict)
if portStr := os.Getenv("PROXY_API_PORT"); portStr != "" {
_, _ = fmt.Sscanf(portStr, "%d", &apiPort)
}
return NewProxyNotificationInjector(apiPort), nil
}
// ProxyNotificationInjector implements NotificationInjector using cae-resp-proxy
type ProxyNotificationInjector struct {
apiPort int
apiBaseURL string
cmd *exec.Cmd
httpClient *http.Client
nodes []proxyNode
visibleNodes []int // Indices of nodes visible in CLUSTER SLOTS (for migration simulation)
}
type proxyNode struct {
listenPort int
targetHost string
targetPort int
proxyAddr string
nodeID string
}
// NewProxyNotificationInjector creates a new proxy-based notification injector
func NewProxyNotificationInjector(apiPort int) *ProxyNotificationInjector {
return &ProxyNotificationInjector{
apiPort: apiPort,
apiBaseURL: fmt.Sprintf("http://localhost:%d", apiPort),
httpClient: &http.Client{
Timeout: 5 * time.Second,
},
nodes: make([]proxyNode, 0),
}
}
func (p *ProxyNotificationInjector) Start() error {
// Get cluster configuration from environment
clusterAddrs := os.Getenv("CLUSTER_ADDRS")
if clusterAddrs == "" {
// Start with 4 nodes: 17000, 17001, 17002, 17003
// Initially, CLUSTER SLOTS will only expose 17000, 17001, 17002
// Node 17003 will be "hidden" until SMIGRATED swaps it in for 17002
clusterAddrs = "127.0.0.1:17000,127.0.0.1:17001,127.0.0.1:17002,127.0.0.1:17003" // Use 127.0.0.1 to force IPv4
}
targetHost := os.Getenv("REDIS_TARGET_HOST")
if targetHost == "" {
targetHost = "127.0.0.1" // Use 127.0.0.1 to force IPv4
}
targetPort := 6379
if portStr := os.Getenv("REDIS_TARGET_PORT"); portStr != "" {
_, _ = fmt.Sscanf(portStr, "%d", &targetPort)
}
// Parse cluster addresses
addrs := strings.Split(clusterAddrs, ",")
if len(addrs) == 0 {
return fmt.Errorf("no cluster addresses specified")
}
// Extract first port for initial node
var initialPort int
_, _ = fmt.Sscanf(strings.Split(addrs[0], ":")[1], "%d", &initialPort)
// Check if proxy is already running (e.g., in Docker)
proxyAlreadyRunning := false
resp, err := p.httpClient.Get(p.apiBaseURL + "/stats")
if err == nil && resp.StatusCode == 200 {
resp.Body.Close()
proxyAlreadyRunning = true
fmt.Printf("✓ Detected existing proxy at %s (e.g., Docker container)\n", p.apiBaseURL)
}
// Only start proxy if not already running
if !proxyAlreadyRunning {
// Start proxy with initial node
p.cmd = exec.Command("cae-resp-proxy",
"--api-port", fmt.Sprintf("%d", p.apiPort),
"--listen-port", fmt.Sprintf("%d", initialPort),
"--target-host", targetHost,
"--target-port", fmt.Sprintf("%d", targetPort),
)
if err := p.cmd.Start(); err != nil {
return fmt.Errorf("failed to start proxy: %w", err)
}
// Wait for proxy to be ready
time.Sleep(500 * time.Millisecond)
}
for i := 0; i < 10; i++ {
resp, err := p.httpClient.Get(p.apiBaseURL + "/stats")
if err == nil {
resp.Body.Close()
if resp.StatusCode == 200 {
// Add all nodes from the cluster addresses
// For Docker proxy, all nodes are already running
// For local proxy, we'll add them via API
for i, addr := range addrs {
var port int
_, _ = fmt.Sscanf(strings.Split(addr, ":")[1], "%d", &port)
if i == 0 {
// Add initial node directly
p.nodes = append(p.nodes, proxyNode{
listenPort: port,
targetHost: targetHost,
targetPort: targetPort,
proxyAddr: fmt.Sprintf("127.0.0.1:%d", port),
nodeID: fmt.Sprintf("127.0.0.1:%d:%d", port, targetPort),
})
} else if !proxyAlreadyRunning {
// Add remaining nodes via API (only if we started the proxy ourselves)
if err := p.addNode(port, targetPort, targetHost); err != nil {
return fmt.Errorf("failed to add node %d: %w", i, err)
}
} else {
// Docker proxy: nodes are already running, just add to our list
p.nodes = append(p.nodes, proxyNode{
listenPort: port,
targetHost: targetHost,
targetPort: targetPort,
proxyAddr: fmt.Sprintf("127.0.0.1:%d", port),
nodeID: fmt.Sprintf("127.0.0.1:%d:%d", port, targetPort),
})
}
}
// Initially, make only the first 3 nodes visible in CLUSTER SLOTS
// The 4th node (index 3) will be hidden until SMIGRATED swaps it in
if len(p.nodes) >= 4 {
p.visibleNodes = []int{0, 1, 2} // Nodes 17000, 17001, 17002
} else {
// If we have fewer than 4 nodes, make all visible
p.visibleNodes = make([]int, len(p.nodes))
for i := range p.visibleNodes {
p.visibleNodes[i] = i
}
}
// Add cluster command interceptors to make standalone Redis appear as a cluster
if err := p.setupClusterInterceptors(); err != nil {
return fmt.Errorf("failed to setup cluster interceptors: %w", err)
}
return nil
}
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("proxy did not become ready")
}
func (p *ProxyNotificationInjector) addNode(listenPort, targetPort int, targetHost string) error {
nodeConfig := map[string]interface{}{
"listenPort": listenPort,
"targetHost": targetHost,
"targetPort": targetPort,
}
jsonData, err := json.Marshal(nodeConfig)
if err != nil {
return fmt.Errorf("failed to marshal node config: %w", err)
}
resp, err := p.httpClient.Post(
p.apiBaseURL+"/nodes",
"application/json",
bytes.NewReader(jsonData),
)
if err != nil {
return fmt.Errorf("failed to add node: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("failed to add node, status %d: %s", resp.StatusCode, string(body))
}
p.nodes = append(p.nodes, proxyNode{
listenPort: listenPort,
targetHost: targetHost,
targetPort: targetPort,
proxyAddr: fmt.Sprintf("localhost:%d", listenPort),
nodeID: fmt.Sprintf("localhost:%d:%d", listenPort, targetPort),
})
time.Sleep(200 * time.Millisecond)
return nil
}
func (p *ProxyNotificationInjector) buildClusterSlotsResponse() string {
// Build CLUSTER SLOTS response dynamically based on VISIBLE nodes only
// Format: Array of slot ranges, each containing:
// - start slot (integer)
// - end slot (integer)
// - master node array: [host, port]
// - replica arrays (optional)
// For simplicity, divide slots equally among visible nodes
totalSlots := 16384
visibleCount := len(p.visibleNodes)
if visibleCount == 0 {
visibleCount = len(p.nodes)
}
slotsPerNode := totalSlots / visibleCount
response := fmt.Sprintf("*%d\r\n", visibleCount) // Number of slot ranges
for i, nodeIdx := range p.visibleNodes {
if nodeIdx >= len(p.nodes) {
continue
}
node := p.nodes[nodeIdx]
startSlot := i * slotsPerNode
endSlot := startSlot + slotsPerNode - 1
if i == visibleCount-1 {
endSlot = 16383 // Last node gets remaining slots
}
// Extract host and port from proxyAddr
host, portStr, _ := strings.Cut(node.proxyAddr, ":")
response += "*3\r\n" // 3 elements: start, end, master
response += fmt.Sprintf(":%d\r\n", startSlot)
response += fmt.Sprintf(":%d\r\n", endSlot)
response += "*2\r\n" // master info: 2 elements (host, port)
response += fmt.Sprintf("$%d\r\n%s\r\n", len(host), host)
response += fmt.Sprintf(":%s\r\n", portStr)
}
return response
}
func (p *ProxyNotificationInjector) addClusterSlotsInterceptor() error {
clusterSlotsResponse := p.buildClusterSlotsResponse()
interceptor := map[string]interface{}{
"name": "cluster-slots",
"match": "*2\r\n$7\r\nCLUSTER\r\n$5\r\nSLOTS\r\n",
"response": clusterSlotsResponse,
"encoding": "raw",
}
jsonData, err := json.Marshal(interceptor)
if err != nil {
return fmt.Errorf("failed to marshal interceptor: %w", err)
}
resp, err := p.httpClient.Post(
p.apiBaseURL+"/interceptors",
"application/json",
bytes.NewReader(jsonData),
)
if err != nil {
return fmt.Errorf("failed to add interceptor: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("failed to add interceptor, status %d: %s", resp.StatusCode, string(body))
}
return nil
}
func (p *ProxyNotificationInjector) updateClusterSlotsInterceptor() error {
// The proxy doesn't support updating existing interceptors
// As a workaround, we'll send the updated CLUSTER SLOTS response directly to all clients
// This simulates what would happen when the client calls CLUSTER SLOTS after SMIGRATED
clusterSlotsResponse := p.buildClusterSlotsResponse()
// Send the updated CLUSTER SLOTS response to all connected clients
// This uses the /send-to-all-clients endpoint
payload := map[string]interface{}{
"data": clusterSlotsResponse,
}
jsonData, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("failed to marshal payload: %w", err)
}
resp, err := p.httpClient.Post(
p.apiBaseURL+"/send-to-all-clients",
"application/json",
bytes.NewReader(jsonData),
)
if err != nil {
// If this fails, it's not critical - the client will get the updated
// response when it calls CLUSTER SLOTS after receiving SMIGRATED
return nil
}
defer resp.Body.Close()
return nil
}
func (p *ProxyNotificationInjector) setupClusterInterceptors() error {
// Add CLUSTER SLOTS interceptor
if err := p.addClusterSlotsInterceptor(); err != nil {
return err
}
// Interceptors to add
interceptors := []map[string]interface{}{
{
"name": "client-maint-notifications-on-with-endpoint",
"match": "*5\r\n$6\r\nclient\r\n$19\r\nmaint_notifications\r\n$2\r\non\r\n$21\r\nmoving-endpoint-type\r\n$4\r\nnone\r\n",
"response": "+OK\r\n",
"encoding": "raw",
},
{
"name": "client-maint-notifications-off",
"match": "*3\r\n$6\r\nclient\r\n$19\r\nmaint_notifications\r\n$3\r\noff\r\n",
"response": "+OK\r\n",
"encoding": "raw",
},
}
// Add all interceptors
for _, interceptor := range interceptors {
jsonData, err := json.Marshal(interceptor)
if err != nil {
return fmt.Errorf("failed to marshal interceptor %s: %w", interceptor["name"], err)
}
resp, err := p.httpClient.Post(
p.apiBaseURL+"/interceptors",
"application/json",
bytes.NewReader(jsonData),
)
if err != nil {
return fmt.Errorf("failed to add interceptor %s: %w", interceptor["name"], err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("failed to add interceptor %s, status %d: %s", interceptor["name"], resp.StatusCode, string(body))
}
fmt.Printf("✓ Added %s interceptor to proxy\n", interceptor["name"])
}
return nil
}
func (p *ProxyNotificationInjector) Stop() error {
if p.cmd != nil && p.cmd.Process != nil {
return p.cmd.Process.Kill()
}
return nil
}
func (p *ProxyNotificationInjector) GetClusterAddrs() []string {
addrs := make([]string, len(p.nodes))
for i, node := range p.nodes {
addrs[i] = node.proxyAddr
}
return addrs
}
func (p *ProxyNotificationInjector) IsReal() bool {
return false
}
func (p *ProxyNotificationInjector) GetTestModeConfig() *TestModeConfig {
return &TestModeConfig{
Mode: TestModeProxyMock,
NotificationDelay: 1 * time.Second,
ActionWaitTimeout: 10 * time.Second,
ActionPollInterval: 500 * time.Millisecond,
DatabaseReadyDelay: 1 * time.Second,
ConnectionEstablishDelay: 500 * time.Millisecond,
MaxClients: 1,
SkipMultiClientTests: true,
}
}
func (p *ProxyNotificationInjector) InjectSMIGRATING(ctx context.Context, seqID int64, slots ...string) error {
notification := formatSMigratingNotification(seqID, slots...)
return p.injectNotification(notification)
}
func (p *ProxyNotificationInjector) InjectSMIGRATED(ctx context.Context, seqID int64, hostPort string, slots ...string) error {
// Simulate topology change by swapping visible nodes
// If we have 4 nodes and currently showing [0,1,2], swap to [0,1,3]
// This simulates node 2 (17002) being replaced by node 3 (17003)
if len(p.nodes) >= 4 && len(p.visibleNodes) == 3 {
// Check if the hostPort matches node 3 (the "new" node)
if hostPort == p.nodes[3].proxyAddr {
// Swap node 2 for node 3 in visible nodes
p.visibleNodes = []int{0, 1, 3}
// Update CLUSTER SLOTS interceptor to reflect new topology
if err := p.updateClusterSlotsInterceptor(); err != nil {
return fmt.Errorf("failed to update CLUSTER SLOTS after migration: %w", err)
}
}
}
// Format endpoint as "host:port slot1,slot2,range1-range2"
endpoint := fmt.Sprintf("%s %s", hostPort, strings.Join(slots, ","))
notification := formatSMigratedNotification(seqID, endpoint)
return p.injectNotification(notification)
}
func (p *ProxyNotificationInjector) InjectMOVING(ctx context.Context, seqID int64, timeS int64, endpoint string) error {
notification := formatMovingNotification(seqID, timeS, endpoint)
return p.injectNotification(notification)
}
func (p *ProxyNotificationInjector) InjectMIGRATING(ctx context.Context, seqID int64, slot int) error {
notification := formatMigratingNotification(seqID, slot)
return p.injectNotification(notification)
}
func (p *ProxyNotificationInjector) InjectMIGRATED(ctx context.Context, seqID int64, slot int) error {
notification := formatMigratedNotification(seqID, slot)
return p.injectNotification(notification)
}
func (p *ProxyNotificationInjector) InjectFAILING_OVER(ctx context.Context, seqID int64) error {
notification := formatFailingOverNotification(seqID)
return p.injectNotification(notification)
}
func (p *ProxyNotificationInjector) InjectFAILED_OVER(ctx context.Context, seqID int64) error {
notification := formatFailedOverNotification(seqID)
return p.injectNotification(notification)
}
func (p *ProxyNotificationInjector) injectNotification(notification string) error {
url := p.apiBaseURL + "/send-to-all-clients?encoding=raw"
resp, err := p.httpClient.Post(url, "application/octet-stream", strings.NewReader(notification))
if err != nil {
return fmt.Errorf("failed to inject notification: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("injection failed with status %d: %s", resp.StatusCode, string(body))
}
return nil
}
// Helper functions to format notifications
func formatMovingNotification(seqID int64, timeS int64, endpoint string) string {
// Format: ["MOVING", seqID, timeS, endpoint]
if endpoint == "" {
// 3 elements: MOVING, seqID, timeS
return fmt.Sprintf(">3\r\n$6\r\nMOVING\r\n:%d\r\n:%d\r\n", seqID, timeS)
}
// 4 elements: MOVING, seqID, timeS, endpoint
return fmt.Sprintf(">4\r\n$6\r\nMOVING\r\n:%d\r\n:%d\r\n$%d\r\n%s\r\n", seqID, timeS, len(endpoint), endpoint)
}
func formatMigratingNotification(seqID int64, slot int) string {
slotStr := fmt.Sprintf("%d", slot)
return fmt.Sprintf(">3\r\n$9\r\nMIGRATING\r\n:%d\r\n$%d\r\n%s\r\n", seqID, len(slotStr), slotStr)
}
func formatMigratedNotification(seqID int64, slot int) string {
slotStr := fmt.Sprintf("%d", slot)
return fmt.Sprintf(">3\r\n$8\r\nMIGRATED\r\n:%d\r\n$%d\r\n%s\r\n", seqID, len(slotStr), slotStr)
}
func formatFailingOverNotification(seqID int64) string {
// Format: ["FAILING_OVER", seqID]
return fmt.Sprintf(">2\r\n$12\r\nFAILING_OVER\r\n:%d\r\n", seqID)
}
func formatFailedOverNotification(seqID int64) string {
// Format: ["FAILED_OVER", seqID]
return fmt.Sprintf(">2\r\n$11\r\nFAILED_OVER\r\n:%d\r\n", seqID)
}
// FaultInjectorNotificationInjector implements NotificationInjector using the real fault injector
type FaultInjectorNotificationInjector struct {
client *FaultInjectorClient
clusterAddrs []string
bdbID int
}
// NewFaultInjectorNotificationInjector creates a new fault injector-based notification injector
func NewFaultInjectorNotificationInjector(baseURL string) *FaultInjectorNotificationInjector {
// Get cluster addresses from environment
clusterAddrs := os.Getenv("CLUSTER_ADDRS")
if clusterAddrs == "" {
clusterAddrs = "localhost:6379"
}
bdbID := 1
if bdbIDStr := os.Getenv("BDB_ID"); bdbIDStr != "" {
_, _ = fmt.Sscanf(bdbIDStr, "%d", &bdbID)
}
return &FaultInjectorNotificationInjector{
client: NewFaultInjectorClient(baseURL),
clusterAddrs: strings.Split(clusterAddrs, ","),
bdbID: bdbID,
}
}
func (f *FaultInjectorNotificationInjector) Start() error {
// Fault injector is already running, nothing to start
return nil
}
func (f *FaultInjectorNotificationInjector) Stop() error {
// Fault injector keeps running, nothing to stop
return nil
}
func (f *FaultInjectorNotificationInjector) GetClusterAddrs() []string {
return f.clusterAddrs
}
func (f *FaultInjectorNotificationInjector) IsReal() bool {
return true
}
func (f *FaultInjectorNotificationInjector) GetTestModeConfig() *TestModeConfig {
return &TestModeConfig{
Mode: TestModeRealFaultInjector,
NotificationDelay: 30 * time.Second,
ActionWaitTimeout: 5 * time.Minute, // Real fault injector can take up to 5 minutes
ActionPollInterval: 500 * time.Millisecond,
DatabaseReadyDelay: 10 * time.Second,
ConnectionEstablishDelay: 2 * time.Second,
MaxClients: 3,
SkipMultiClientTests: false,
}
}
func (f *FaultInjectorNotificationInjector) InjectSMIGRATING(ctx context.Context, seqID int64, slots ...string) error {
// For real fault injector, we trigger actual slot migration which will generate SMIGRATING
// Parse slot ranges
var startSlot, endSlot int
if len(slots) > 0 {
if strings.Contains(slots[0], "-") {
_, _ = fmt.Sscanf(slots[0], "%d-%d", &startSlot, &endSlot)
} else {
_, _ = fmt.Sscanf(slots[0], "%d", &startSlot)
endSlot = startSlot
}
}
// Trigger slot migration (this will generate SMIGRATING notification)
resp, err := f.client.TriggerSlotMigration(ctx, startSlot, endSlot, "node-1", "node-2")
if err != nil {
return fmt.Errorf("failed to trigger slot migration: %w", err)
}
// Wait for action to start
_, err = f.client.WaitForAction(ctx, resp.ActionID, WithMaxWaitTime(10*time.Second))
return err
}
func (f *FaultInjectorNotificationInjector) InjectSMIGRATED(ctx context.Context, seqID int64, hostPort string, slots ...string) error {
// SMIGRATED is generated automatically when migration completes
// We can't directly inject it with the real fault injector
// This is a limitation of using the real fault injector
return fmt.Errorf("SMIGRATED cannot be directly injected with real fault injector - it's generated when migration completes")
}
func (f *FaultInjectorNotificationInjector) InjectMOVING(ctx context.Context, seqID int64, timeS int64, endpoint string) error {
// MOVING notifications are generated during bind action
return fmt.Errorf("MOVING cannot be directly injected with real fault injector - it's generated during bind action")
}
func (f *FaultInjectorNotificationInjector) InjectMIGRATING(ctx context.Context, seqID int64, slot int) error {
// Trigger slot migration for standalone
resp, err := f.client.TriggerSlotMigration(ctx, slot, slot, "node-1", "node-2")
if err != nil {
return fmt.Errorf("failed to trigger slot migration: %w", err)
}
_, err = f.client.WaitForAction(ctx, resp.ActionID, WithMaxWaitTime(10*time.Second))
return err
}
func (f *FaultInjectorNotificationInjector) InjectMIGRATED(ctx context.Context, seqID int64, slot int) error {
// MIGRATED is generated automatically when migration completes
return fmt.Errorf("MIGRATED cannot be directly injected with real fault injector - it's generated when migration completes")
}
func (f *FaultInjectorNotificationInjector) InjectFAILING_OVER(ctx context.Context, seqID int64) error {
// FAILING_OVER is generated automatically when failover starts
return fmt.Errorf("FAILING_OVER cannot be directly injected with real fault injector - it's generated when failover starts")
}
func (f *FaultInjectorNotificationInjector) InjectFAILED_OVER(ctx context.Context, seqID int64) error {
// FAILED_OVER is generated automatically when failover completes
return fmt.Errorf("FAILED_OVER cannot be directly injected with real fault injector - it's generated when failover completes")
}
maintnotifications/e2e/notiftracker_test.go
+249 -95
View File
@@ -23,6 +23,10 @@ type DiagnosticsEvent struct {
ConnID uint64 `json:"connID"`
SeqID int64 `json:"seqID"`
// ShardAddr is the address of the shard that received this notification
// Only populated when using NewTrackingNotificationsHookWithShard
ShardAddr string `json:"shardAddr,omitempty"`
Error error `json:"error"`
Pre bool `json:"pre"`
@@ -45,24 +49,58 @@ type TrackingNotificationsHook struct {
totalNotifications atomic.Int64
migratingCount atomic.Int64
migratedCount atomic.Int64
sMigratingCount atomic.Int64
sMigratedCount atomic.Int64
failingOverCount atomic.Int64
failedOverCount atomic.Int64
movingCount atomic.Int64
unexpectedNotificationCount atomic.Int64
// cluster state reload tracking (actual reloads, not just SMIGRATED notifications)
clusterStateReloadCount atomic.Int64
diagnosticsLog []DiagnosticsEvent
connIds map[uint64]bool
connLogs map[uint64][]DiagnosticsEvent
mutex sync.RWMutex
// shard identifier
shardAddr string
// track last notification time for waiting
lastNotificationTime atomic.Value // stores time.Time
}
// NewTrackingNotificationsHook creates a new notification hook with counters
func NewTrackingNotificationsHook() *TrackingNotificationsHook {
return &TrackingNotificationsHook{
hook := &TrackingNotificationsHook{
diagnosticsLog: make([]DiagnosticsEvent, 0),
connIds: make(map[uint64]bool),
connLogs: make(map[uint64][]DiagnosticsEvent),
}
hook.lastNotificationTime.Store(time.Time{})
return hook
}
// NewTrackingNotificationsHookWithShard creates a hook with shard identifier
func NewTrackingNotificationsHookWithShard(shardAddr string) *TrackingNotificationsHook {
hook := &TrackingNotificationsHook{
diagnosticsLog: make([]DiagnosticsEvent, 0),
connIds: make(map[uint64]bool),
connLogs: make(map[uint64][]DiagnosticsEvent),
shardAddr: shardAddr,
}
hook.lastNotificationTime.Store(time.Time{})
return hook
}
// SetShardAddr sets the shard address
func (tnh *TrackingNotificationsHook) SetShardAddr(addr string) {
tnh.shardAddr = addr
}
// GetLastNotificationTime returns the time of the last notification received
func (tnh *TrackingNotificationsHook) GetLastNotificationTime() time.Time {
return tnh.lastNotificationTime.Load().(time.Time)
}
// it is not reusable, but just to keep it consistent
@@ -79,7 +117,14 @@ func (tnh *TrackingNotificationsHook) Clear() {
tnh.totalNotifications.Store(0)
tnh.migratingCount.Store(0)
tnh.migratedCount.Store(0)
tnh.sMigratingCount.Store(0)
tnh.sMigratedCount.Store(0)
tnh.failingOverCount.Store(0)
tnh.failedOverCount.Store(0)
tnh.movingCount.Store(0)
tnh.unexpectedNotificationCount.Store(0)
tnh.connectionCount.Store(0)
tnh.clusterStateReloadCount.Store(0)
}
// wait for notification in prehook
@@ -116,6 +161,9 @@ func (tnh *TrackingNotificationsHook) FindNotification(notificationType string)
// PreHook captures timeout-related events before processing
func (tnh *TrackingNotificationsHook) PreHook(_ context.Context, notificationCtx push.NotificationHandlerContext, notificationType string, notification []interface{}) ([]interface{}, bool) {
// Update last notification time
tnh.lastNotificationTime.Store(time.Now())
tnh.increaseNotificationCount(notificationType)
tnh.storeDiagnosticsEvent(notificationType, notification, notificationCtx)
tnh.increaseRelaxedTimeoutCount(notificationType)
@@ -161,10 +209,18 @@ func (tnh *TrackingNotificationsHook) PostHook(_ context.Context, notificationCt
func (tnh *TrackingNotificationsHook) storeDiagnosticsEvent(notificationType string, notification []interface{}, notificationCtx push.NotificationHandlerContext) {
connID := tnh.getConnID(notificationCtx)
// Get shard address - prefer explicit shardAddr, fall back to connection's remote address
shardAddr := tnh.shardAddr
if shardAddr == "" {
shardAddr = tnh.getShardAddrFromContext(notificationCtx)
}
event := DiagnosticsEvent{
Type: notificationType,
ConnID: connID,
SeqID: tnh.getSeqID(notification),
ShardAddr: shardAddr,
Pre: true,
Timestamp: time.Now(),
Details: map[string]interface{}{
@@ -183,6 +239,16 @@ func (tnh *TrackingNotificationsHook) storeDiagnosticsEvent(notificationType str
tnh.mutex.Unlock()
}
// getShardAddrFromContext extracts the shard address from the notification context
func (tnh *TrackingNotificationsHook) getShardAddrFromContext(notificationCtx push.NotificationHandlerContext) string {
if conn, ok := notificationCtx.Conn.(*pool.Conn); ok {
if remoteAddr := conn.RemoteAddr(); remoteAddr != nil {
return remoteAddr.String()
}
}
return ""
}
// GetRelaxedTimeoutCount returns the count of relaxed timeout events
func (tnh *TrackingNotificationsHook) GetRelaxedTimeoutCount() int64 {
return tnh.relaxedTimeoutCount.Load()
@@ -232,6 +298,10 @@ func (tnh *TrackingNotificationsHook) increaseNotificationCount(notificationType
tnh.migratingCount.Add(1)
case "MIGRATED":
tnh.migratedCount.Add(1)
case "SMIGRATING":
tnh.sMigratingCount.Add(1)
case "SMIGRATED":
tnh.sMigratedCount.Add(1)
case "FAILING_OVER":
tnh.failingOverCount.Add(1)
case "FAILED_OVER":
@@ -243,83 +313,13 @@ func (tnh *TrackingNotificationsHook) increaseNotificationCount(notificationType
func (tnh *TrackingNotificationsHook) increaseRelaxedTimeoutCount(notificationType string) {
switch notificationType {
case "MIGRATING", "FAILING_OVER":
case "MIGRATING", "SMIGRATING", "FAILING_OVER":
tnh.relaxedTimeoutCount.Add(1)
case "MIGRATED", "FAILED_OVER":
case "MIGRATED", "SMIGRATED", "FAILED_OVER":
tnh.unrelaxedTimeoutCount.Add(1)
}
}
// setupNotificationHook sets up tracking for both regular and cluster clients with notification hooks
func setupNotificationHook(client redis.UniversalClient, hook maintnotifications.NotificationHook) {
if clusterClient, ok := client.(*redis.ClusterClient); ok {
setupClusterClientNotificationHook(clusterClient, hook)
} else if regularClient, ok := client.(*redis.Client); ok {
setupRegularClientNotificationHook(regularClient, hook)
}
}
// setupNotificationHooks sets up tracking for both regular and cluster clients with notification hooks
func setupNotificationHooks(client redis.UniversalClient, hooks ...maintnotifications.NotificationHook) {
for _, hook := range hooks {
setupNotificationHook(client, hook)
}
}
// setupRegularClientNotificationHook sets up notification hook for regular clients
func setupRegularClientNotificationHook(client *redis.Client, hook maintnotifications.NotificationHook) {
maintnotificationsManager := client.GetMaintNotificationsManager()
if maintnotificationsManager != nil {
maintnotificationsManager.AddNotificationHook(hook)
} else {
fmt.Printf("[TNH] Warning: Maintenance notifications manager not available for tracking\n")
}
}
// setupClusterClientNotificationHook sets up notification hook for cluster clients
func setupClusterClientNotificationHook(client *redis.ClusterClient, hook maintnotifications.NotificationHook) {
ctx := context.Background()
// Register hook on existing nodes
err := client.ForEachShard(ctx, func(ctx context.Context, nodeClient *redis.Client) error {
maintnotificationsManager := nodeClient.GetMaintNotificationsManager()
if maintnotificationsManager != nil {
maintnotificationsManager.AddNotificationHook(hook)
} else {
fmt.Printf("[TNH] Warning: Maintenance notifications manager not available for tracking on node: %s\n", nodeClient.Options().Addr)
}
return nil
})
if err != nil {
fmt.Printf("[TNH] Warning: Failed to register timeout tracking hooks on existing cluster nodes: %v\n", err)
}
// Register hook on new nodes
client.OnNewNode(func(nodeClient *redis.Client) {
maintnotificationsManager := nodeClient.GetMaintNotificationsManager()
if maintnotificationsManager != nil {
maintnotificationsManager.AddNotificationHook(hook)
} else {
fmt.Printf("[TNH] Warning: Maintenance notifications manager not available for tracking on new node: %s\n", nodeClient.Options().Addr)
}
})
}
// filterPushNotificationLogs filters the diagnostics log for push notification events
func filterPushNotificationLogs(diagnosticsLog []DiagnosticsEvent) []DiagnosticsEvent {
var pushNotificationLogs []DiagnosticsEvent
for _, log := range diagnosticsLog {
switch log.Type {
case "MOVING", "MIGRATING", "MIGRATED":
pushNotificationLogs = append(pushNotificationLogs, log)
}
}
return pushNotificationLogs
}
func (tnh *TrackingNotificationsHook) GetAnalysis() *DiagnosticsAnalysis {
return NewDiagnosticsAnalysis(tnh.GetDiagnosticsLog())
}
@@ -349,13 +349,16 @@ type DiagnosticsAnalysis struct {
MovingCount int64
MigratingCount int64
MigratedCount int64
SMigratingCount int64
SMigratedCount int64
FailingOverCount int64
FailedOverCount int64
UnexpectedNotificationCount int64
TotalNotifications int64
diagnosticsLog []DiagnosticsEvent
connLogs map[uint64][]DiagnosticsEvent
connIds map[uint64]bool
diagnosticsLog []DiagnosticsEvent
connLogs map[uint64][]DiagnosticsEvent
connIds map[uint64]bool
}
func NewDiagnosticsAnalysis(diagnosticsLog []DiagnosticsEvent) *DiagnosticsAnalysis {
@@ -369,46 +372,61 @@ func NewDiagnosticsAnalysis(diagnosticsLog []DiagnosticsEvent) *DiagnosticsAnaly
return da
}
func (da *DiagnosticsAnalysis) Analyze() {
for _, log := range da.diagnosticsLog {
da.TotalNotifications++
func (a *DiagnosticsAnalysis) Analyze() {
for _, log := range a.diagnosticsLog {
a.TotalNotifications++
switch log.Type {
case "MOVING":
da.MovingCount++
a.MovingCount++
case "MIGRATING":
da.MigratingCount++
a.MigratingCount++
case "MIGRATED":
da.MigratedCount++
a.MigratedCount++
case "SMIGRATING":
a.SMigratingCount++
case "SMIGRATED":
a.SMigratedCount++
case "FAILING_OVER":
da.FailingOverCount++
a.FailingOverCount++
case "FAILED_OVER":
da.FailedOverCount++
a.FailedOverCount++
default:
da.UnexpectedNotificationCount++
a.UnexpectedNotificationCount++
}
if log.Error != nil {
fmt.Printf("[ERROR] Notification processing error: %v\n", log.Error)
fmt.Printf("[ERROR] Notification: %v\n", log.Details["notification"])
fmt.Printf("[ERROR] Context: %v\n", log.Details["context"])
da.NotificationProcessingErrors++
a.NotificationProcessingErrors++
}
if log.Type == "MIGRATING" || log.Type == "FAILING_OVER" {
da.RelaxedTimeoutCount++
} else if log.Type == "MIGRATED" || log.Type == "FAILED_OVER" {
da.UnrelaxedTimeoutCount++
switch log.Type {
case "MIGRATING", "SMIGRATING", "FAILING_OVER":
a.RelaxedTimeoutCount++
case "MIGRATED", "SMIGRATED", "FAILED_OVER":
a.UnrelaxedTimeoutCount++
}
if log.ConnID != 0 {
if v, ok := da.connIds[log.ConnID]; !ok || !v {
da.connIds[log.ConnID] = true
da.connLogs[log.ConnID] = make([]DiagnosticsEvent, 0)
da.ConnectionCount++
if v, ok := a.connIds[log.ConnID]; !ok || !v {
a.connIds[log.ConnID] = true
a.connLogs[log.ConnID] = make([]DiagnosticsEvent, 0)
a.ConnectionCount++
}
da.connLogs[log.ConnID] = append(da.connLogs[log.ConnID], log)
a.connLogs[log.ConnID] = append(a.connLogs[log.ConnID], log)
}
}
}
// PrintSummary prints a compact summary without detailed per-event information
func (a *DiagnosticsAnalysis) PrintSummary(t *testing.T) {
t.Logf("Notification Summary: %d events, %d connections | MOVING:%d MIGRATING:%d MIGRATED:%d FAILING_OVER:%d FAILED_OVER:%d SMIGRATING:%d SMIGRATED:%d | Errors:%d",
len(a.diagnosticsLog), len(a.connIds),
a.MovingCount, a.MigratingCount, a.MigratedCount,
a.FailingOverCount, a.FailedOverCount,
a.SMigratingCount, a.SMigratedCount,
a.NotificationProcessingErrors)
}
func (a *DiagnosticsAnalysis) Print(t *testing.T) {
t.Logf("Notification Analysis results for %d events and %d connections:", len(a.diagnosticsLog), len(a.connIds))
t.Logf("-------------")
@@ -428,9 +446,145 @@ func (a *DiagnosticsAnalysis) Print(t *testing.T) {
t.Logf(" - FAILED_OVER: %d", a.FailedOverCount)
t.Logf(" - Unexpected: %d", a.UnexpectedNotificationCount)
t.Logf("-------------")
t.Logf("- CLUSTER-SPECIFIC Notification Analysis-")
t.Logf("-------------")
t.Logf(" - SMIGRATING: %d", a.SMigratingCount)
t.Logf(" - SMIGRATED: %d", a.SMigratedCount)
t.Logf("-------------")
t.Logf(" - Total Notifications: %d", a.TotalNotifications)
t.Logf(" - Notification Processing Errors: %d", a.NotificationProcessingErrors)
t.Logf(" - Connection Count: %d", a.ConnectionCount)
t.Logf("-------------")
// Print detailed notification events grouped by seqID and type
t.Logf("-Detailed Notification Events (grouped by seqID)-")
t.Logf("-------------")
// Group events by (seqID, type)
type eventKey struct {
seqID int64
typ string
}
type shardEvent struct {
shardAddr string
connID uint64
timestamp time.Time
notification []interface{}
}
groupedEvents := make(map[eventKey][]shardEvent)
var seqIDOrder []eventKey // Track order of first occurrence
for _, event := range a.diagnosticsLog {
if !event.Pre {
continue
}
key := eventKey{seqID: event.SeqID, typ: event.Type}
if _, exists := groupedEvents[key]; !exists {
seqIDOrder = append(seqIDOrder, key)
}
// Extract notification from details
var notification []interface{}
if event.Details != nil {
if notif, ok := event.Details["notification"].([]interface{}); ok {
notification = notif
}
}
groupedEvents[key] = append(groupedEvents[key], shardEvent{
shardAddr: event.ShardAddr,
connID: event.ConnID,
timestamp: event.Timestamp,
notification: notification,
})
}
// Print grouped events
for _, key := range seqIDOrder {
events := groupedEvents[key]
t.Logf(" seqID=%d type=%s (received on %d shard(s)):", key.seqID, key.typ, len(events))
// Print notification content once (should be same for all shards)
if len(events) > 0 && events[0].notification != nil {
t.Logf(" notification: %v", events[0].notification)
}
// Print which shards received it
for _, se := range events {
shardInfo := se.shardAddr
if shardInfo == "" {
shardInfo = "unknown"
}
t.Logf(" - shard=%s connID=%d time=%s",
shardInfo, se.connID, se.timestamp.Format("15:04:05.000"))
}
}
t.Logf("-------------")
t.Logf("Diagnostics Analysis completed successfully")
}
// setupNotificationHook adds a notification hook to a cluster client
//
//nolint:unused // Used in test files
func setupNotificationHook(client *redis.ClusterClient, hook maintnotifications.NotificationHook) {
_ = client.ForEachShard(context.Background(), func(ctx context.Context, nodeClient *redis.Client) error {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.AddNotificationHook(hook)
}
return nil
})
// Also add hook to new nodes
client.OnNewNode(func(nodeClient *redis.Client) {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.AddNotificationHook(hook)
}
})
}
// setupNotificationHooks adds multiple notification hooks to a regular client
//
//nolint:unused // Used in test files
func setupNotificationHooks(client redis.UniversalClient, hooks ...maintnotifications.NotificationHook) {
// Try to get manager from the client
var manager *maintnotifications.Manager
// Check if it's a regular client
if regularClient, ok := client.(*redis.Client); ok {
manager = regularClient.GetMaintNotificationsManager()
}
// Check if it's a cluster client
if clusterClient, ok := client.(*redis.ClusterClient); ok {
// For cluster clients, add hooks to all shards
_ = clusterClient.ForEachShard(context.Background(), func(ctx context.Context, nodeClient *redis.Client) error {
nodeManager := nodeClient.GetMaintNotificationsManager()
if nodeManager != nil {
for _, hook := range hooks {
nodeManager.AddNotificationHook(hook)
}
}
return nil
})
// Also add hooks to new nodes
clusterClient.OnNewNode(func(nodeClient *redis.Client) {
nodeManager := nodeClient.GetMaintNotificationsManager()
if nodeManager != nil {
for _, hook := range hooks {
nodeManager.AddNotificationHook(hook)
}
}
})
return
}
// For regular clients, add hooks directly
if manager != nil {
for _, hook := range hooks {
manager.AddNotificationHook(hook)
}
}
}
maintnotifications/e2e/proxy_fault_injector_server.go
+1480
View File
File diff suppressed because it is too large Load Diff
maintnotifications/e2e/scenario_endpoint_types_test.go
+29 -9
View File
@@ -58,15 +58,21 @@ func TestEndpointTypesPushNotifications(t *testing.T) {
for _, endpointTest := range endpointTypes {
t.Run(endpointTest.name, func(t *testing.T) {
// Setup: Create fresh database and client factory for THIS endpoint type test
bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
bdbID, factory, testMode, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
defer cleanup()
t.Logf("[ENDPOINT-TYPES-%s] Created test database with bdb_id: %d", endpointTest.name, bdbID)
t.Logf("[ENDPOINT-TYPES-%s] Created test database with bdb_id: %d (mode: %s)", endpointTest.name, bdbID, testMode.Mode)
// Create fault injector
faultInjector, err := CreateTestFaultInjector()
// Skip this test if using proxy mock (requires real fault injector)
if testMode.IsProxyMock() {
t.Skip("Skipping endpoint type test - requires real fault injector")
}
// Create fault injector with cleanup
faultInjector, fiCleanup, err := CreateTestFaultInjectorWithCleanup()
if err != nil {
t.Fatalf("[ERROR] Failed to create fault injector: %v", err)
}
defer fiCleanup()
// Get endpoint config from factory (now connected to new database)
endpointConfig := factory.GetConfig()
@@ -245,10 +251,22 @@ func TestEndpointTypesPushNotifications(t *testing.T) {
p("MIGRATED notification received for %s. %v", endpointTest.name, migratedData)
// Complete migration with bind action
// Pass endpoint_type to the bind action so it knows what format to use
var endpointTypeStr string
switch endpointTest.endpointType {
case maintnotifications.EndpointTypeExternalIP:
endpointTypeStr = "external-ip"
case maintnotifications.EndpointTypeExternalFQDN:
endpointTypeStr = "external-fqdn"
case maintnotifications.EndpointTypeNone:
endpointTypeStr = "none"
}
bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{
Type: "bind",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
"bdb_id": endpointConfig.BdbID,
"endpoint_type": endpointTypeStr,
},
})
if err != nil {
@@ -287,12 +305,14 @@ func TestEndpointTypesPushNotifications(t *testing.T) {
var expectedAddress string
hostParts := strings.SplitN(endpointConfig.Host, ".", 2)
if len(hostParts) != 2 {
e("invalid host %s", endpointConfig.Host)
// Docker proxy setup uses "localhost" without domain suffix
// In this case, skip FQDN validation
p("Skipping FQDN validation for Docker proxy setup (host=%s)", endpointConfig.Host)
} else {
expectedAddress = hostParts[1]
}
if address != expectedAddress {
e("invalid fqdn, expected: %s, got: %s", expectedAddress, address)
if address != expectedAddress {
e("invalid fqdn, expected: %s, got: %s", expectedAddress, address)
}
}
case maintnotifications.EndpointTypeExternalIP:
maintnotifications/e2e/scenario_push_notifications_test.go
+372 -207
View File
@@ -13,7 +13,8 @@ import (
"github.com/redis/go-redis/v9/maintnotifications"
)
// TestPushNotifications tests Redis Enterprise push notifications (MOVING, MIGRATING, MIGRATED)
// TestPushNotifications tests Redis Enterprise push notifications (MOVING, MIGRATING, MIGRATED, FAILING_OVER, FAILED_OVER)
// This test now works with BOTH the real fault injector and the proxy mock
func TestPushNotifications(t *testing.T) {
if os.Getenv("E2E_SCENARIO_TESTS") != "true" {
t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true")
@@ -23,12 +24,12 @@ func TestPushNotifications(t *testing.T) {
defer cancel()
// Setup: Create fresh database and client factory for this test
bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
bdbID, factory, testMode, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
defer cleanup()
t.Logf("[PUSH-NOTIFICATIONS] Created test database with bdb_id: %d", bdbID)
t.Logf("[PUSH-NOTIFICATIONS] Created test database with bdb_id: %d (mode: %s)", bdbID, testMode.Mode)
// Wait for database to be fully ready
time.Sleep(10 * time.Second)
// Wait for database to be fully ready (mode-aware)
time.Sleep(testMode.DatabaseReadyDelay)
var dump = true
var seqIDToObserve int64
@@ -38,6 +39,9 @@ func TestPushNotifications(t *testing.T) {
var found bool
var status *ActionStatusResponse
var bindStatus *ActionStatusResponse
var movingNotification []interface{}
var commandsRunner2, commandsRunner3 *CommandRunner
var errorsDetected = false
var p = func(format string, args ...interface{}) {
@@ -62,10 +66,20 @@ func TestPushNotifications(t *testing.T) {
// Get endpoint config from factory (now connected to new database)
endpointConfig := factory.GetConfig()
// Create fault injector
faultInjector, err := CreateTestFaultInjector()
// Create notification injector (works with both proxy mock and real FI)
injector, err := NewNotificationInjector()
if err != nil {
ef("Failed to create fault injector: %v", err)
ef("Failed to create notification injector: %v", err)
}
defer injector.Stop()
// For real fault injector, we also need the FaultInjectorClient for actions
var faultInjector *FaultInjectorClient
if !testMode.IsProxyMock() {
faultInjector, err = CreateTestFaultInjector()
if err != nil {
ef("Failed to create fault injector: %v", err)
}
}
minIdleConns := 5
@@ -123,16 +137,28 @@ func TestPushNotifications(t *testing.T) {
}()
p("Starting FAILING_OVER / FAILED_OVER notifications test...")
// Test: Trigger failover action to generate FAILING_OVER, FAILED_OVER notifications
p("Triggering failover action to generate push notifications...")
failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{
Type: "failover",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger failover action: %v", err)
// Mode-aware: Proxy mock directly injects notifications, real FI triggers actions
var failoverResp *ActionResponse
if testMode.IsProxyMock() {
// Proxy mock: Directly inject FAILING_OVER notification
p("Injecting FAILING_OVER notification (proxy mock mode)...")
if err := injector.InjectFAILING_OVER(ctx, 1000); err != nil {
ef("Failed to inject FAILING_OVER: %v", err)
}
time.Sleep(testMode.NotificationDelay)
} else {
// Real FI: Trigger failover action to generate FAILING_OVER, FAILED_OVER notifications
p("Triggering failover action to generate push notifications...")
failoverResp, err = faultInjector.TriggerAction(ctx, ActionRequest{
Type: "failover",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger failover action: %v", err)
}
}
go func() {
p("Waiting for FAILING_OVER notification")
@@ -149,10 +175,22 @@ func TestPushNotifications(t *testing.T) {
p("FAILING_OVER notification received. %v", failingOverData)
seqIDToObserve = int64(failingOverData["seqID"].(float64))
connIDToObserve = uint64(failingOverData["connID"].(float64))
// Inject FAILED_OVER in proxy mock mode
if testMode.IsProxyMock() {
p("Injecting FAILED_OVER notification (proxy mock mode)...")
if err := injector.InjectFAILED_OVER(ctx, 1001); err != nil {
ef("Failed to inject FAILED_OVER: %v", err)
}
time.Sleep(testMode.NotificationDelay)
}
go func() {
p("Waiting for FAILED_OVER notification on conn %d with seqID %d...", connIDToObserve, seqIDToObserve+1)
// Note: Redis Enterprise may not send FAILED_OVER with seqID = FAILING_OVER.seqID + 1
// We wait for any FAILED_OVER on the same connection
p("Waiting for FAILED_OVER notification on conn %d...", connIDToObserve)
match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool {
return notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1)
return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "FAILED_OVER") && connID(s, connIDToObserve)
}, 3*time.Minute)
commandsRunner.Stop()
}()
@@ -163,36 +201,92 @@ func TestPushNotifications(t *testing.T) {
failedOverData := logs2.ExtractDataFromLogMessage(match)
p("FAILED_OVER notification received. %v", failedOverData)
status, err = faultInjector.WaitForAction(ctx, failoverResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Failover action failed: %v", err)
// Wait for action to complete (real FI only)
if !testMode.IsProxyMock() {
status, err = faultInjector.WaitForAction(ctx, failoverResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Failover action failed: %v", err)
}
p("Failover action completed: %s %s", status.Status, actionOutputIfFailed(status))
}
p("[FI] Failover action completed: %v %s", status.Status, actionOutputIfFailed(status))
p("FAILING_OVER / FAILED_OVER notifications test completed successfully")
// Test: Trigger migrate action to generate MOVING, MIGRATING, MIGRATED notifications
p("Triggering migrate action to generate push notifications...")
migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{
Type: "migrate",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger migrate action: %v", err)
var migrateResp *ActionResponse
if testMode.IsProxyMock() {
// Proxy mock: Directly inject MIGRATING notification
p("Injecting MIGRATING notification (proxy mock mode)...")
if err := injector.InjectMIGRATING(ctx, 2000, 5000); err != nil {
ef("Failed to inject MIGRATING: %v", err)
}
time.Sleep(testMode.NotificationDelay)
} else {
// Real FI: Trigger migrate action
p("Triggering migrate action to generate push notifications...")
migrateResp, err = faultInjector.TriggerAction(ctx, ActionRequest{
Type: "migrate",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger migrate action: %v", err)
}
}
go func() {
if DebugE2E() {
fmt.Printf("[DEBUG] Goroutine started: waiting for MIGRATING notification...\n")
}
match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool {
return strings.Contains(s, logs2.ProcessingNotificationMessage) && strings.Contains(s, "MIGRATING")
return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MIGRATING")
}, 60*time.Second)
if DebugE2E() {
fmt.Printf("[DEBUG] Goroutine: MatchOrWaitForLogMatchFunc returned, found=%v, calling Stop()...\n", found)
}
commandsRunner.Stop()
if DebugE2E() {
fmt.Printf("[DEBUG] Goroutine: Stop() returned\n")
}
}()
if DebugE2E() {
fmt.Printf("[DEBUG] Main thread: calling FireCommandsUntilStop...\n")
}
commandsRunner.FireCommandsUntilStop(ctx)
if DebugE2E() {
fmt.Printf("[DEBUG] Main thread: FireCommandsUntilStop returned\n")
}
if !found {
if !testMode.IsProxyMock() {
status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Migrate action failed: %v", err)
}
p("[FI] Migrate action completed: %s %s", status.Status, actionOutputIfFailed(status))
}
ef("MIGRATING notification for migrate action was not received within 60 seconds")
}
migrateData := logs2.ExtractDataFromLogMessage(match)
seqIDToObserve = int64(migrateData["seqID"].(float64))
connIDToObserve = uint64(migrateData["connID"].(float64))
p("MIGRATING notification received: seqID: %d, connID: %d", seqIDToObserve, connIDToObserve)
// Wait for action to complete and inject MIGRATED (mode-aware)
if testMode.IsProxyMock() {
// Proxy mock: Directly inject MIGRATED notification
p("Injecting MIGRATED notification (proxy mock mode)...")
if err := injector.InjectMIGRATED(ctx, 2001, 5000); err != nil {
ef("Failed to inject MIGRATED: %v", err)
}
time.Sleep(testMode.NotificationDelay)
} else {
// Real FI: Wait for action to complete (generates MIGRATED automatically)
status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
@@ -201,26 +295,15 @@ func TestPushNotifications(t *testing.T) {
ef("[FI] Migrate action failed: %v", err)
}
p("[FI] Migrate action completed: %s %s", status.Status, actionOutputIfFailed(status))
ef("MIGRATING notification for migrate action was not received within 60 seconds")
}
migrateData := logs2.ExtractDataFromLogMessage(match)
seqIDToObserve = int64(migrateData["seqID"].(float64))
connIDToObserve = uint64(migrateData["connID"].(float64))
p("MIGRATING notification received: seqID: %d, connID: %d", seqIDToObserve, connIDToObserve)
status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Migrate action failed: %v", err)
}
p("[FI] Migrate action completed: %s %s", status.Status, actionOutputIfFailed(status))
go func() {
p("Waiting for MIGRATED notification on conn %d with seqID %d...", connIDToObserve, seqIDToObserve+1)
// Note: During a real migration, the connection that received MIGRATING might be closed.
// MIGRATED might arrive on a different connection.
// We accept any MIGRATED notification rather than requiring same connection/seqID.
p("Waiting for MIGRATED notification (preferring conn %d with seqID %d)...", connIDToObserve, seqIDToObserve+1)
match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool {
return notificationType(s, "MIGRATED") && connID(s, connIDToObserve) && seqID(s, seqIDToObserve+1)
return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MIGRATED")
}, 3*time.Minute)
commandsRunner.Stop()
}()
@@ -233,169 +316,236 @@ func TestPushNotifications(t *testing.T) {
p("MIGRATING / MIGRATED notifications test completed successfully")
// Trigger bind action to complete the migration process
p("Triggering bind action to complete migration...")
// Trigger bind action to complete the migration process (or inject MOVING in proxy mock mode)
var bindResp *ActionResponse
if testMode.IsProxyMock() {
// Proxy mock: We need connections to receive the MOVING notification,
// but we must stop traffic before handoffs start to avoid data race.
// The handoff worker reinitializes connections, which races with CommandRunner.
p("Starting commands before MOVING injection (proxy mock mode)...")
go commandsRunner.FireCommandsUntilStop(ctx)
// Give commands time to establish connections
time.Sleep(1 * time.Second)
bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{
Type: "bind",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger bind action: %v", err)
}
// Proxy mock: Directly inject MOVING notification
// Format: ["MOVING", seqID, timeS, endpoint]
// timeS is the time in seconds until the connection should be handed off
p("Injecting MOVING notification (proxy mock mode)...")
if err := injector.InjectMOVING(ctx, 3000, 30, ""); err != nil {
ef("Failed to inject MOVING: %v", err)
}
time.Sleep(testMode.NotificationDelay)
// start a second client but don't execute any commands on it
p("Starting a second client to observe notification during moving...")
client2, err := factory.Create("push-notification-client-2", &CreateClientOptions{
Protocol: 3, // RESP3 required for push notifications
PoolSize: poolSize,
MinIdleConns: minIdleConns,
MaxActiveConns: maxConnections,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeEnabled,
HandoffTimeout: 40 * time.Second, // 30 seconds
RelaxedTimeout: 30 * time.Minute, // 30 minutes relaxed timeout for second client
PostHandoffRelaxedDuration: 2 * time.Second, // 2 seconds post-handoff relaxed duration
MaxWorkers: 20,
EndpointType: maintnotifications.EndpointTypeExternalIP, // Use external IP for enterprise
},
ClientName: "push-notification-test-client-2",
})
if err != nil {
ef("failed to create client: %v", err)
}
// setup tracking for second client
tracker2 := NewTrackingNotificationsHook()
logger2 := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug))
setupNotificationHooks(client2, tracker2, logger2)
commandsRunner2, _ := NewCommandRunner(client2)
p("Second client created")
// Use a channel to communicate errors from the goroutine
errChan := make(chan error, 1)
go func() {
defer func() {
if r := recover(); r != nil {
errChan <- fmt.Errorf("goroutine panic: %v", r)
}
}()
p("Waiting for MOVING notification on first client")
match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool {
// Wait for MOVING notification to be received before stopping CommandRunner
p("Waiting for MOVING notification to be received (proxy mock mode)...")
match, found := logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool {
return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING")
}, 3*time.Minute)
commandsRunner.Stop()
}, 30*time.Second)
if !found {
errChan <- fmt.Errorf("MOVING notification was not received within 3 minutes ON A FIRST CLIENT")
return
ef("MOVING notification was not received in proxy mock mode")
}
movingData := logs2.ExtractDataFromLogMessage(match)
p("MOVING notification received (proxy mock mode). %v", movingData)
// once moving is received, start a second client commands runner
p("Starting commands on second client")
go commandsRunner2.FireCommandsUntilStop(ctx)
p("Waiting for MOVING notification on second client")
matchNotif, fnd := tracker2.FindOrWaitForNotification("MOVING", 3*time.Minute)
if !fnd {
errChan <- fmt.Errorf("MOVING notification was not received within 3 minutes ON A SECOND CLIENT")
return
} else {
p("MOVING notification received on second client %v", matchNotif)
}
// Signal success
errChan <- nil
}()
commandsRunner.FireCommandsUntilStop(ctx)
// wait for moving on first client
// once the commandRunner stops, it means a waiting
// on the logCollector match has completed and we can proceed
if !found {
ef("MOVING notification was not received within 3 minutes")
}
movingData := logs2.ExtractDataFromLogMessage(match)
p("MOVING notification received. %v", movingData)
seqIDToObserve = int64(movingData["seqID"].(float64))
connIDToObserve = uint64(movingData["connID"].(float64))
time.Sleep(3 * time.Second)
// start a third client but don't execute any commands on it
p("Starting a third client to observe notification during moving...")
client3, err := factory.Create("push-notification-client-2", &CreateClientOptions{
Protocol: 3, // RESP3 required for push notifications
PoolSize: poolSize,
MinIdleConns: minIdleConns,
MaxActiveConns: maxConnections,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeEnabled,
HandoffTimeout: 40 * time.Second, // 30 seconds
RelaxedTimeout: 30 * time.Minute, // 30 minutes relaxed timeout for second client
PostHandoffRelaxedDuration: 2 * time.Second, // 2 seconds post-handoff relaxed duration
MaxWorkers: 20,
EndpointType: maintnotifications.EndpointTypeExternalIP, // Use external IP for enterprise
},
ClientName: "push-notification-test-client-3",
})
if err != nil {
ef("failed to create client: %v", err)
}
// setup tracking for second client
tracker3 := NewTrackingNotificationsHook()
logger3 := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug))
setupNotificationHooks(client3, tracker3, logger3)
commandsRunner3, _ := NewCommandRunner(client3)
p("Third client created")
go commandsRunner3.FireCommandsUntilStop(ctx)
// wait for moving on third client
movingNotification, found := tracker3.FindOrWaitForNotification("MOVING", 3*time.Minute)
if !found {
p("[NOTICE] MOVING notification was not received within 3 minutes ON A THIRD CLIENT")
// Stop command runner before handoffs start to avoid data race
// Handoffs are scheduled at timeS/2 = 15 seconds, so stop before that
p("Stopping command runner before handoffs start (proxy mock mode)...")
commandsRunner.Stop()
} else {
p("MOVING notification received on third client. %v", movingNotification)
if len(movingNotification) != 4 {
p("[NOTICE] Invalid MOVING notification format: %s", movingNotification)
}
mNotifTimeS, ok := movingNotification[2].(int64)
if !ok {
p("[NOTICE] Invalid timeS in MOVING notification: %s", movingNotification)
}
// expect timeS to be less than 15
if mNotifTimeS < 15 {
p("[NOTICE] Expected timeS < 15, got %d", mNotifTimeS)
// Real FI: Trigger bind action
p("Triggering bind action to complete migration...")
bindResp, err = faultInjector.TriggerAction(ctx, ActionRequest{
Type: "bind",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger bind action: %v", err)
}
}
commandsRunner3.Stop()
// Wait for the goroutine to complete and check for errors
if err := <-errChan; err != nil {
ef("Second client goroutine error: %v", err)
// Multi-client tests - only run if not using proxy mock
if !testMode.SkipMultiClientTests {
// start a second client but don't execute any commands on it
p("Starting a second client to observe notification during moving...")
client2, err := factory.Create("push-notification-client-2", &CreateClientOptions{
Protocol: 3, // RESP3 required for push notifications
PoolSize: poolSize,
MinIdleConns: minIdleConns,
MaxActiveConns: maxConnections,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeEnabled,
HandoffTimeout: 40 * time.Second, // 30 seconds
RelaxedTimeout: 30 * time.Minute, // 30 minutes relaxed timeout for second client
PostHandoffRelaxedDuration: 2 * time.Second, // 2 seconds post-handoff relaxed duration
MaxWorkers: 20,
EndpointType: maintnotifications.EndpointTypeExternalIP, // Use external IP for enterprise
},
ClientName: "push-notification-test-client-2",
})
if err != nil {
ef("failed to create client: %v", err)
}
// setup tracking for second client
tracker2 := NewTrackingNotificationsHook()
logger2 := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug))
setupNotificationHooks(client2, tracker2, logger2)
commandsRunner2, _ = NewCommandRunner(client2)
p("Second client created")
// Use a channel to communicate errors from the goroutine
errChan := make(chan error, 1)
go func() {
defer func() {
if r := recover(); r != nil {
errChan <- fmt.Errorf("goroutine panic: %v", r)
}
}()
p("Waiting for MOVING notification on first client")
match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool {
return strings.Contains(s, logs2.ProcessingNotificationMessage) && notificationType(s, "MOVING")
}, 3*time.Minute)
commandsRunner.Stop()
if !found {
errChan <- fmt.Errorf("MOVING notification was not received within 3 minutes ON A FIRST CLIENT")
return
}
// once moving is received, start a second client commands runner
p("Starting commands on second client")
go commandsRunner2.FireCommandsUntilStop(ctx)
p("Waiting for MOVING notification on second client")
matchNotif, fnd := tracker2.FindOrWaitForNotification("MOVING", 3*time.Minute)
if !fnd {
errChan <- fmt.Errorf("MOVING notification was not received within 3 minutes ON A SECOND CLIENT")
return
} else {
p("MOVING notification received on second client %v", matchNotif)
}
// Signal success
errChan <- nil
}()
commandsRunner.FireCommandsUntilStop(ctx)
// wait for moving on first client
// once the commandRunner stops, it means a waiting
// on the logCollector match has completed and we can proceed
if !found {
ef("MOVING notification was not received within 3 minutes")
}
movingData := logs2.ExtractDataFromLogMessage(match)
p("MOVING notification received. %v", movingData)
seqIDToObserve = int64(movingData["seqID"].(float64))
connIDToObserve = uint64(movingData["connID"].(float64))
time.Sleep(3 * time.Second)
// start a third client but don't execute any commands on it
p("Starting a third client to observe notification during moving...")
client3, err := factory.Create("push-notification-test-client-3", &CreateClientOptions{
Protocol: 3, // RESP3 required for push notifications
PoolSize: poolSize,
MinIdleConns: minIdleConns,
MaxActiveConns: maxConnections,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeEnabled,
HandoffTimeout: 40 * time.Second, // 30 seconds
RelaxedTimeout: 30 * time.Minute, // 30 minutes relaxed timeout for second client
PostHandoffRelaxedDuration: 2 * time.Second, // 2 seconds post-handoff relaxed duration
MaxWorkers: 20,
EndpointType: maintnotifications.EndpointTypeExternalIP, // Use external IP for enterprise
},
ClientName: "push-notification-test-client-3",
})
if err != nil {
ef("failed to create client: %v", err)
}
// setup tracking for third client
tracker3 := NewTrackingNotificationsHook()
logger3 := maintnotifications.NewLoggingHook(int(logging.LogLevelDebug))
setupNotificationHooks(client3, tracker3, logger3)
commandsRunner3, _ = NewCommandRunner(client3)
p("Third client created")
go commandsRunner3.FireCommandsUntilStop(ctx)
// wait for moving on third client
movingNotification, found = tracker3.FindOrWaitForNotification("MOVING", 3*time.Minute)
if !found {
p("[NOTICE] MOVING notification was not received within 3 minutes ON A THIRD CLIENT")
} else {
p("MOVING notification received on third client. %v", movingNotification)
if len(movingNotification) != 4 {
p("[NOTICE] Invalid MOVING notification format: %s", movingNotification)
}
mNotifTimeS, ok := movingNotification[2].(int64)
if !ok {
p("[NOTICE] Invalid timeS in MOVING notification: %s", movingNotification)
}
// expect timeS to be less than 15
if mNotifTimeS < 15 {
p("[NOTICE] Expected timeS < 15, got %d", mNotifTimeS)
}
}
commandsRunner3.Stop()
commandsRunner2.Stop() // Stop commandsRunner2 before the final section
// Wait for the goroutine to complete and check for errors
if err := <-errChan; err != nil {
ef("Second client goroutine error: %v", err)
}
// Wait for bind action to complete
bindStatus, err = faultInjector.WaitForAction(ctx, bindResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second))
if err != nil {
ef("Bind action failed: %v", err)
}
p("Bind action completed: %s %s", bindStatus.Status, actionOutputIfFailed(bindStatus))
p("MOVING notification test completed successfully")
} else {
p("Skipping multi-client MOVING test (proxy mock mode)")
}
// Wait for bind action to complete
bindStatus, err := faultInjector.WaitForAction(ctx, bindResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second))
if err != nil {
ef("Bind action failed: %v", err)
}
p("Bind action completed: %s %s", bindStatus.Status, actionOutputIfFailed(bindStatus))
p("MOVING notification test completed successfully")
p("Executing commands and collecting logs for analysis... ")
go commandsRunner.FireCommandsUntilStop(ctx)
go commandsRunner2.FireCommandsUntilStop(ctx)
go commandsRunner3.FireCommandsUntilStop(ctx)
time.Sleep(2 * time.Minute)
commandsRunner.Stop()
commandsRunner2.Stop()
commandsRunner3.Stop()
time.Sleep(1 * time.Minute)
// Run commands based on mode
if testMode.SkipMultiClientTests {
// Single client mode (proxy mock)
// CommandRunner was already stopped before handoffs started (to avoid data race)
// Wait for handoffs to complete:
// 1. MOVING notification was already processed
// 2. Handoff is scheduled at timeS/2 = 15 seconds
// 3. Handoff executes
// Total: ~18 seconds for handoffs to complete
p("Waiting for handoffs to complete (proxy mock mode)...")
time.Sleep(18 * time.Second)
// Restart command runner to observe post-handoff behavior
p("Restarting command runner to observe post-handoff behavior...")
commandsRunner, _ = NewCommandRunner(client)
go commandsRunner.FireCommandsUntilStop(ctx)
// Run traffic for a bit to observe post-handoff behavior
time.Sleep(5 * time.Second)
commandsRunner.Stop()
} else {
// Multi-client mode (real FI)
go commandsRunner.FireCommandsUntilStop(ctx)
go commandsRunner2.FireCommandsUntilStop(ctx)
go commandsRunner3.FireCommandsUntilStop(ctx)
time.Sleep(2 * time.Minute)
commandsRunner.Stop()
commandsRunner2.Stop()
commandsRunner3.Stop()
}
time.Sleep(2 * testMode.NotificationDelay)
allLogsAnalysis := logCollector.GetAnalysis()
trackerAnalysis := tracker.GetAnalysis()
@@ -421,9 +571,24 @@ func TestPushNotifications(t *testing.T) {
e("Expected relaxed timeouts after post-handoff, got none")
}
// validate number of connections we do not exceed max connections
// we started three clients, so we expect 3x the connections
if allLogsAnalysis.ConnectionCount > int64(maxConnections)*3 {
e("Expected no more than %d connections, got %d", maxConnections*3, allLogsAnalysis.ConnectionCount)
// Adjust expected connections based on mode
// During handoffs, new connections are created before old ones are closed,
// so the connection count can temporarily exceed the pool size.
// We use a multiplier to account for this.
expectedMaxConns := int64(maxConnections)
if !testMode.SkipMultiClientTests {
// We started three clients, and during handoffs each connection can temporarily
// have both old and new connections active. We allow 4x the base connections
// to account for handoff overhead across all clients.
expectedMaxConns = int64(maxConnections) * 4
} else {
// In proxy mock mode, we test with a standalone client,
// During handoffs, new connections are created before old ones are closed,
expectedMaxConns = int64(maxConnections) * 2
}
if allLogsAnalysis.ConnectionCount > expectedMaxConns {
e("Expected no more than %d connections, got %d", expectedMaxConns, allLogsAnalysis.ConnectionCount)
}
if allLogsAnalysis.ConnectionCount < int64(minIdleConns) {
maintnotifications/e2e/scenario_stress_test.go
+13 -8
View File
@@ -23,12 +23,17 @@ func TestStressPushNotifications(t *testing.T) {
defer cancel()
// Setup: Create fresh database and client factory for this test
bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
bdbID, factory, testMode, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
defer cleanup()
t.Logf("[STRESS] Created test database with bdb_id: %d", bdbID)
t.Logf("[STRESS] Created test database with bdb_id: %d (mode: %s)", bdbID, testMode.Mode)
// Wait for database to be fully ready
time.Sleep(10 * time.Second)
// Skip this test if using proxy mock (stress test requires real fault injector)
if testMode.IsProxyMock() {
t.Skip("Skipping stress test - requires real fault injector")
}
// Wait for database to be fully ready (mode-aware)
time.Sleep(testMode.DatabaseReadyDelay)
var dump = true
var errorsDetected = false
@@ -302,10 +307,10 @@ func TestStressPushNotifications(t *testing.T) {
p("Error rate: %.2f%%, Notification processing errors: %d/%d",
errorRate, totalProcessingErrors, totalTrackerNotifications)
// Print final analysis
allLogsAnalysis.Print(t)
// Print final analysis (summary only for stress tests to reduce output)
allLogsAnalysis.PrintSummary(t)
for i, tracker := range trackers {
p("=== Stress Client %d Analysis ===", i)
tracker.GetAnalysis().Print(t)
p("=== Stress Client %d ===", i)
tracker.GetAnalysis().PrintSummary(t)
}
}
maintnotifications/e2e/scenario_timeout_configs_test.go
+166 -63
View File
@@ -14,6 +14,7 @@ import (
)
// TestTimeoutConfigurationsPushNotifications tests push notifications with different timeout configurations
// This test now works with BOTH the real fault injector and the proxy mock
func TestTimeoutConfigurationsPushNotifications(t *testing.T) {
if os.Getenv("E2E_SCENARIO_TESTS") != "true" {
t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true")
@@ -78,17 +79,27 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) {
for _, timeoutTest := range timeoutConfigs {
t.Run(timeoutTest.name, func(t *testing.T) {
// Setup: Create fresh database and client factory for THIS timeout config test
bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
bdbID, factory, testMode, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
defer cleanup()
t.Logf("[TIMEOUT-CONFIGS-%s] Created test database with bdb_id: %d", timeoutTest.name, bdbID)
t.Logf("[TIMEOUT-CONFIGS-%s] Created test database with bdb_id: %d (mode: %s)", timeoutTest.name, bdbID, testMode.Mode)
// Get endpoint config from factory (now connected to new database)
endpointConfig := factory.GetConfig()
// Create fault injector
faultInjector, err := CreateTestFaultInjector()
// Create notification injector (works with both proxy mock and real FI)
injector, err := NewNotificationInjector()
if err != nil {
t.Fatalf("[ERROR] Failed to create fault injector: %v", err)
t.Fatalf("[ERROR] Failed to create notification injector: %v", err)
}
defer injector.Stop()
// For real fault injector, we also need the FaultInjectorClient for actions
var faultInjector *FaultInjectorClient
if !testMode.IsProxyMock() {
faultInjector, err = CreateTestFaultInjector()
if err != nil {
t.Fatalf("[ERROR] Failed to create fault injector: %v", err)
}
}
defer func() {
@@ -169,14 +180,25 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) {
// Test failover with this timeout configuration
p("Testing failover with %s timeout configuration...", timeoutTest.name)
failoverResp, err := faultInjector.TriggerAction(ctx, ActionRequest{
Type: "failover",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger failover action for %s: %v", timeoutTest.name, err)
var failoverResp *ActionResponse
if testMode.IsProxyMock() {
// Proxy mock: Directly inject FAILING_OVER notification
p("Injecting FAILING_OVER notification (proxy mock mode)...")
if err := injector.InjectFAILING_OVER(ctx, 1000); err != nil {
ef("Failed to inject FAILING_OVER: %v", err)
}
time.Sleep(testMode.NotificationDelay)
} else {
// Real FI: Trigger failover action
failoverResp, err = faultInjector.TriggerAction(ctx, ActionRequest{
Type: "failover",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger failover action for %s: %v", timeoutTest.name, err)
}
}
// Wait for FAILING_OVER notification
@@ -189,6 +211,15 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) {
failingOverData := logs2.ExtractDataFromLogMessage(match)
p("FAILING_OVER notification received for %s. %v", timeoutTest.name, failingOverData)
// Inject FAILED_OVER in proxy mock mode
if testMode.IsProxyMock() {
p("Injecting FAILED_OVER notification (proxy mock mode)...")
if err := injector.InjectFAILED_OVER(ctx, 1001); err != nil {
ef("Failed to inject FAILED_OVER: %v", err)
}
time.Sleep(testMode.NotificationDelay)
}
// Wait for FAILED_OVER notification
seqIDToObserve := int64(failingOverData["seqID"].(float64))
connIDToObserve := uint64(failingOverData["connID"].(float64))
@@ -201,42 +232,62 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) {
failedOverData := logs2.ExtractDataFromLogMessage(match)
p("FAILED_OVER notification received for %s. %v", timeoutTest.name, failedOverData)
// Wait for failover to complete
status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID,
WithMaxWaitTime(180*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Failover action failed for %s: %v", timeoutTest.name, err)
// Wait for failover to complete (real FI only)
if !testMode.IsProxyMock() {
status, err := faultInjector.WaitForAction(ctx, failoverResp.ActionID,
WithMaxWaitTime(180*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Failover action failed for %s: %v", timeoutTest.name, err)
}
p("[FI] Failover action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status))
}
p("[FI] Failover action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status))
// Continue traffic to observe timeout behavior
p("Continuing traffic for %v to observe timeout behavior...", timeoutTest.relaxedTimeout*2)
time.Sleep(timeoutTest.relaxedTimeout * 2)
// In proxy mock mode, use shorter sleep since notifications are immediate
// but still need enough time to observe timeout behavior
trafficObservationDuration := timeoutTest.relaxedTimeout * 2
if testMode.IsProxyMock() {
// Use 11 seconds for proxy mock - enough to observe timeout behavior
trafficObservationDuration = 11 * time.Second
}
p("Continuing traffic for %v to observe timeout behavior...", trafficObservationDuration)
time.Sleep(trafficObservationDuration)
// Test migration to trigger more timeout scenarios
p("Testing migration with %s timeout configuration...", timeoutTest.name)
migrateResp, err := faultInjector.TriggerAction(ctx, ActionRequest{
Type: "migrate",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger migrate action for %s: %v", timeoutTest.name, err)
}
var migrateResp *ActionResponse
if testMode.IsProxyMock() {
// Proxy mock: Directly inject MIGRATING notification
p("Injecting MIGRATING notification (proxy mock mode)...")
if err := injector.InjectMIGRATING(ctx, 2000, 5000); err != nil {
ef("Failed to inject MIGRATING: %v", err)
}
time.Sleep(testMode.NotificationDelay)
} else {
// Real FI: Trigger migrate action
migrateResp, err = faultInjector.TriggerAction(ctx, ActionRequest{
Type: "migrate",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger migrate action for %s: %v", timeoutTest.name, err)
}
// Wait for migration to complete
status, err = faultInjector.WaitForAction(ctx, migrateResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err)
}
// Wait for migration to complete
status, err := faultInjector.WaitForAction(ctx, migrateResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Migrate action failed for %s: %v", timeoutTest.name, err)
}
p("[FI] Migrate action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status))
p("[FI] Migrate action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status))
}
// Wait for MIGRATING notification
match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool {
@@ -248,24 +299,45 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) {
migrateData := logs2.ExtractDataFromLogMessage(match)
p("MIGRATING notification received for %s: %v", timeoutTest.name, migrateData)
// do a bind action
bindResp, err := faultInjector.TriggerAction(ctx, ActionRequest{
Type: "bind",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger bind action for %s: %v", timeoutTest.name, err)
// Inject MIGRATED in proxy mock mode
if testMode.IsProxyMock() {
p("Injecting MIGRATED notification (proxy mock mode)...")
if err := injector.InjectMIGRATED(ctx, 2001, 5000); err != nil {
ef("Failed to inject MIGRATED: %v", err)
}
time.Sleep(testMode.NotificationDelay)
}
status, err = faultInjector.WaitForAction(ctx, bindResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Bind action failed for %s: %v", timeoutTest.name, err)
// do a bind action (or inject MOVING in proxy mock mode)
var bindResp *ActionResponse
if testMode.IsProxyMock() {
// Proxy mock: Directly inject MOVING notification
// CommandRunner is still running so connections can receive the notification
p("Injecting MOVING notification (proxy mock mode)...")
if err := injector.InjectMOVING(ctx, 3000, 30, ""); err != nil {
ef("Failed to inject MOVING: %v", err)
}
time.Sleep(testMode.NotificationDelay)
} else {
// Real FI: Trigger bind action
bindResp, err = faultInjector.TriggerAction(ctx, ActionRequest{
Type: "bind",
Parameters: map[string]interface{}{
"bdb_id": endpointConfig.BdbID,
},
})
if err != nil {
ef("Failed to trigger bind action for %s: %v", timeoutTest.name, err)
}
status, err := faultInjector.WaitForAction(ctx, bindResp.ActionID,
WithMaxWaitTime(240*time.Second),
WithPollInterval(2*time.Second),
)
if err != nil {
ef("[FI] Bind action failed for %s: %v", timeoutTest.name, err)
}
p("[FI] Bind action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status))
}
p("[FI] Bind action completed for %s: %s %s", timeoutTest.name, status.Status, actionOutputIfFailed(status))
// waiting for moving notification
match, found = logCollector.MatchOrWaitForLogMatchFunc(func(s string) bool {
@@ -279,8 +351,32 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) {
p("MOVING notification received for %s. %v", timeoutTest.name, movingData)
// Continue traffic for post-handoff timeout observation
p("Continuing traffic for %v to observe post-handoff timeout behavior...", 1*time.Minute)
time.Sleep(1 * time.Minute)
// In proxy mock mode, use shorter sleep since notifications are immediate
// but still need enough time to observe post-handoff timeout behavior
postHandoffDuration := 1 * time.Minute
if testMode.IsProxyMock() {
// In proxy mock mode, stop the CommandRunner after MOVING is received
// but before handoffs execute to avoid data race.
// The handoff worker reinitializes connections, which races with CommandRunner
// using the same connections.
// Handoffs are scheduled at timeS/2 = 15 seconds.
p("Stopping command runner before handoffs execute (proxy mock mode)...")
commandsRunner.Stop()
// Wait for handoffs to complete
p("Waiting for handoffs to complete...")
time.Sleep(18 * time.Second) // Wait for handoffs (scheduled at 15s)
// Restart command runner to observe post-handoff behavior
p("Restarting command runner to observe post-handoff behavior...")
commandsRunner, _ = NewCommandRunner(client)
go commandsRunner.FireCommandsUntilStop(ctx)
// Use shorter observation time since we already waited for handoffs
postHandoffDuration = 5 * time.Second
}
p("Continuing traffic for %v to observe post-handoff timeout behavior...", postHandoffDuration)
time.Sleep(postHandoffDuration)
commandsRunner.Stop()
testDuration := time.Since(testStartTime)
@@ -295,16 +391,23 @@ func TestTimeoutConfigurationsPushNotifications(t *testing.T) {
// Validate timeout-specific behavior
switch timeoutTest.name {
case "Conservative":
if trackerAnalysis.UnrelaxedTimeoutCount > trackerAnalysis.RelaxedTimeoutCount {
// In proxy mock mode, the timing is more predictable and we may not see
// the same ratio of relaxed to unrelaxed timeouts as with real FI
if !testMode.IsProxyMock() && trackerAnalysis.UnrelaxedTimeoutCount > trackerAnalysis.RelaxedTimeoutCount {
e("Conservative config should have more relaxed than unrelaxed timeouts, got relaxed=%d, unrelaxed=%d",
trackerAnalysis.RelaxedTimeoutCount, trackerAnalysis.UnrelaxedTimeoutCount)
}
case "Aggressive":
// Aggressive timeouts should complete faster
if testDuration > 5*time.Minute {
e("Aggressive config took too long: %v", testDuration)
// Proxy mock is much faster than real FI, so adjust expectations
maxDuration := 5 * time.Minute
if testMode.IsProxyMock() {
maxDuration = 2 * time.Minute
}
if logAnalysis.TotalHandoffRetries > logAnalysis.TotalHandoffCount {
if testDuration > maxDuration {
e("Aggressive config took too long: %v (max: %v)", testDuration, maxDuration)
}
if !testMode.IsProxyMock() && logAnalysis.TotalHandoffRetries > logAnalysis.TotalHandoffCount {
e("Expect handoff retries since aggressive timeouts are shorter, got %d retries for %d handoffs",
logAnalysis.TotalHandoffRetries, logAnalysis.TotalHandoffCount)
}
maintnotifications/e2e/scenario_tls_configs_test.go
+7 -2
View File
@@ -74,9 +74,14 @@ func TestTLSConfigurationsPushNotifications(t *testing.T) {
for _, tlsTest := range tlsConfigs {
t.Run(tlsTest.name, func(t *testing.T) {
// Setup: Create fresh database and client factory for THIS TLS config test
bdbID, factory, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
bdbID, factory, testMode, cleanup := SetupTestDatabaseAndFactory(t, ctx, "standalone")
defer cleanup()
t.Logf("[TLS-CONFIGS-%s] Created test database with bdb_id: %d", tlsTest.name, bdbID)
t.Logf("[TLS-CONFIGS-%s] Created test database with bdb_id: %d (mode: %s)", tlsTest.name, bdbID, testMode.Mode)
// Skip this test if using proxy mock (requires real fault injector)
if testMode.IsProxyMock() {
t.Skip("Skipping TLS config test - requires real fault injector")
}
// Get endpoint config from factory (now connected to new database)
endpointConfig := factory.GetConfig()
maintnotifications/e2e/scenario_unified_injector_test.go
+434
View File
@@ -0,0 +1,434 @@
package e2e
import (
"context"
"fmt"
"os"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
)
// TestUnifiedInjector_SMIGRATING demonstrates using the unified notification injector
// This test requires the proxy-based mock to directly inject notifications
func TestUnifiedInjector_SMIGRATING(t *testing.T) {
// Skip if using real fault injector (these tests require proxy mock for direct notification injection)
if os.Getenv("FAULT_INJECTOR_URL") != "" {
t.Skip("Skipping unified injector test - requires proxy mock, not real fault injector")
}
ctx := context.Background()
// Create notification injector (automatically chooses based on environment)
injector, err := NewNotificationInjector()
if err != nil {
t.Fatalf("[ERROR] Failed to create notification injector: %v", err)
}
// Start the injector - skip if proxy not available
if err := injector.Start(); err != nil {
t.Skipf("Skipping test - proxy not available: %v", err)
}
defer injector.Stop()
// Get test mode configuration
testMode := injector.GetTestModeConfig()
t.Logf("Using %s mode", testMode.Mode)
t.Logf("Cluster addresses: %v", injector.GetClusterAddrs())
// Create cluster client with maintnotifications enabled
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: injector.GetClusterAddrs(),
Protocol: 3, // RESP3 required for push notifications
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeEnabled,
RelaxedTimeout: 30 * time.Second,
},
})
defer client.Close()
// Set up notification tracking
tracker := NewTrackingNotificationsHook()
setupNotificationHook(client, tracker)
// Verify connection
if err := client.Ping(ctx).Err(); err != nil {
t.Fatalf("[ERROR] Failed to connect to cluster: %v", err)
}
// Perform some operations to establish connections
for i := 0; i < 10; i++ {
if err := client.Set(ctx, fmt.Sprintf("key%d", i), "value", 0).Err(); err != nil {
t.Logf("Warning: Failed to set key: %v", err)
}
}
// Start a blocking operation in background to keep connection active
// This ensures the proxy has an active connection to send notifications to
blockingDone := make(chan error, 1)
go func() {
// BLPOP with 10 second timeout - keeps connection active
_, err := client.BLPop(ctx, 10*time.Second, "notification-test-list").Result()
blockingDone <- err
}()
// Wait for blocking command to start (mode-aware)
time.Sleep(testMode.ConnectionEstablishDelay)
// Inject SMIGRATING notification while connection is active
t.Log("Injecting SMIGRATING notification...")
if err := injector.InjectSMIGRATING(ctx, 12345, "1000-2000", "3000"); err != nil {
t.Fatalf("[ERROR] Failed to inject SMIGRATING: %v", err)
}
// Wait for notification processing (mode-aware)
time.Sleep(testMode.NotificationDelay)
// Verify notification was received
analysis := tracker.GetAnalysis()
if analysis.SMigratingCount == 0 {
t.Errorf("[ERROR] Expected to receive SMIGRATING notification, got 0")
} else {
t.Logf("✓ Received %d SMIGRATING notification(s)", analysis.SMigratingCount)
}
// Verify operations still work (timeouts should be relaxed)
if err := client.Set(ctx, "test-key-during-migration", "value", 0).Err(); err != nil {
t.Errorf("[ERROR] Expected operations to work during migration, got error: %v", err)
}
// Print analysis
analysis.Print(t)
t.Log("✓ SMIGRATING test passed")
}
// TestUnifiedInjector_SMIGRATED demonstrates SMIGRATED notification handling
// This test requires the proxy-based mock to directly inject notifications
func TestUnifiedInjector_SMIGRATED(t *testing.T) {
// Skip if using real fault injector (these tests require proxy mock for direct notification injection)
if os.Getenv("FAULT_INJECTOR_URL") != "" {
t.Skip("Skipping unified injector test - requires proxy mock, not real fault injector")
}
ctx := context.Background()
injector, err := NewNotificationInjector()
if err != nil {
t.Fatalf("[ERROR] Failed to create notification injector: %v", err)
}
// Start the injector - skip if proxy not available
if err := injector.Start(); err != nil {
t.Skipf("Skipping test - proxy not available: %v", err)
}
defer injector.Stop()
// Get test mode configuration
testMode := injector.GetTestModeConfig()
t.Logf("Using %s mode", testMode.Mode)
// Track cluster state reloads
var reloadCount atomic.Int32
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: injector.GetClusterAddrs(),
Protocol: 3,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeEnabled,
RelaxedTimeout: 30 * time.Second,
},
})
defer client.Close()
// Set up notification tracking
tracker := NewTrackingNotificationsHook()
setupNotificationHook(client, tracker)
if err := client.Ping(ctx).Err(); err != nil {
t.Fatalf("[ERROR] Failed to connect: %v", err)
}
// Set up reload callback on existing nodes
client.ForEachShard(ctx, func(ctx context.Context, nodeClient *redis.Client) error {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.SetClusterStateReloadCallback(func(ctx context.Context, hostPort string, slotRanges []string) {
reloadCount.Add(1)
t.Logf("Cluster state reload triggered for %s, slots: %v", hostPort, slotRanges)
})
}
return nil
})
// Set up reload callback for new nodes
client.OnNewNode(func(nodeClient *redis.Client) {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.SetClusterStateReloadCallback(func(ctx context.Context, hostPort string, slotRanges []string) {
reloadCount.Add(1)
t.Logf("Cluster state reload triggered for %s, slots: %v", hostPort, slotRanges)
})
}
})
// Perform some operations to establish connections
for i := 0; i < 10; i++ {
if err := client.Set(ctx, fmt.Sprintf("key%d", i), "value", 0).Err(); err != nil {
t.Logf("Warning: Failed to set key: %v", err)
}
}
initialReloads := reloadCount.Load()
// Start a blocking operation in background to keep connection active
blockingDone := make(chan error, 1)
go func() {
// BLPOP with 10 second timeout - keeps connection active
_, err := client.BLPop(ctx, 10*time.Second, "notification-test-list").Result()
blockingDone <- err
}()
// Wait for blocking command to start (mode-aware)
time.Sleep(testMode.ConnectionEstablishDelay)
// Get all node addresses - needed for both modes
addrs := injector.GetClusterAddrs()
var newNodeAddr string
if len(addrs) >= 4 {
newNodeAddr = addrs[3] // Node 3: 127.0.0.1:17003
} else {
// Fallback to first node if we don't have 4 nodes
newNodeAddr = addrs[0]
}
// Mode-specific behavior for triggering SMIGRATED
if testMode.IsProxyMock() {
// Proxy mock: Directly inject SMIGRATED notification
t.Log("Injecting SMIGRATED notification to swap node 2 for node 3...")
t.Logf("Using new node address: %s", newNodeAddr)
if err := injector.InjectSMIGRATED(ctx, 12346, newNodeAddr, "1000-2000", "3000"); err != nil {
t.Fatalf("[ERROR] Failed to inject SMIGRATED: %v", err)
}
} else {
// Real fault injector: Trigger slot migration which generates both SMIGRATING and SMIGRATED
t.Log("Triggering slot migration (will generate SMIGRATING and SMIGRATED)...")
// First inject SMIGRATING (this triggers the actual migration)
if err := injector.InjectSMIGRATING(ctx, 12345, "1000-2000", "3000"); err != nil {
t.Fatalf("[ERROR] Failed to trigger slot migration: %v", err)
}
// Wait for migration to complete (real FI takes longer)
t.Log("Waiting for migration to complete...")
}
// Wait for notification processing (mode-aware)
time.Sleep(testMode.NotificationDelay)
// Wait for blocking operation to complete
<-blockingDone
// Verify notification was received
analysis := tracker.GetAnalysis()
if testMode.IsProxyMock() {
// Proxy mock: SMIGRATED notifications may not always be received
// because they're sent to all connections, but the client might not be actively
// listening on all of them. This is expected behavior.
if analysis.MigratedCount > 0 {
t.Logf("✓ Received %d SMIGRATED notification(s)", analysis.MigratedCount)
} else {
t.Logf("Note: No SMIGRATED notifications received (expected in proxy mock mode)")
}
} else {
// Real FI: Should receive both SMIGRATING and SMIGRATED
if analysis.MigratingCount == 0 {
t.Errorf("[ERROR] Expected to receive SMIGRATING notification with real FI, got 0")
} else {
t.Logf("✓ Received %d SMIGRATING notification(s)", analysis.MigratingCount)
}
if analysis.MigratedCount > 0 {
t.Logf("✓ Received %d SMIGRATED notification(s)", analysis.MigratedCount)
} else {
t.Logf("Note: SMIGRATED notification not yet received (migration may still be in progress)")
}
}
// Verify cluster state reload callback
// Note: SMIGRATED notifications trigger cluster reload via the endpoint information
// However, in proxy mock mode, the callback might not be triggered because
// the notification is sent to all connections, not just the active one
finalReloads := reloadCount.Load()
if finalReloads > initialReloads {
t.Logf("✓ Cluster state reloaded %d time(s)", finalReloads-initialReloads)
} else {
t.Logf("Note: Cluster state reload callback not triggered (expected in proxy mock mode)")
}
// Verify the client discovered the new node topology
// After SMIGRATED, the client should reload cluster slots
// Give it a moment to process
time.Sleep(2 * time.Second)
// Count how many nodes the client knows about
var nodeAddrsMu sync.Mutex
nodeAddrs := make(map[string]bool)
client.ForEachShard(ctx, func(ctx context.Context, nodeClient *redis.Client) error {
addr := nodeClient.Options().Addr
nodeAddrsMu.Lock()
nodeAddrs[addr] = true
nodeAddrsMu.Unlock()
t.Logf("Client knows about node: %s", addr)
return nil
})
// We should have 3 nodes (0, 1, 3) after the swap
if len(nodeAddrs) < 3 {
t.Logf("Warning: Expected client to discover 3 nodes after SMIGRATED, got %d", len(nodeAddrs))
} else {
t.Logf("✓ Client discovered %d node(s) after SMIGRATED", len(nodeAddrs))
}
// Verify the new node (17003) is in the list
if len(addrs) >= 4 && !nodeAddrs[newNodeAddr] {
t.Logf("Warning: Client did not discover new node %s", newNodeAddr)
} else if len(addrs) >= 4 {
t.Logf("✓ Client discovered new node %s", newNodeAddr)
}
// Verify we can still perform operations after SMIGRATED
// The client should have reloaded cluster topology
successCount := 0
for i := 0; i < 10; i++ {
key := fmt.Sprintf("post-migration-key-%d", i)
if err := client.Set(ctx, key, "value", 0).Err(); err == nil {
successCount++
} else {
t.Logf("Warning: Failed to set key after SMIGRATED: %v", err)
}
}
if successCount < 8 {
t.Errorf("[ERROR] Expected most operations to succeed after SMIGRATED, got %d/10", successCount)
} else {
t.Logf("✓ Successfully performed %d/10 operations after SMIGRATED", successCount)
}
// Print analysis
analysis.Print(t)
t.Log("✓ SMIGRATED test passed")
}
// TestUnifiedInjector_ComplexScenario demonstrates a complex migration scenario
// This test requires the proxy-based mock to directly inject notifications
func TestUnifiedInjector_ComplexScenario(t *testing.T) {
// Skip if using real fault injector (these tests require proxy mock for direct notification injection)
if os.Getenv("FAULT_INJECTOR_URL") != "" {
t.Skip("Skipping unified injector test - requires proxy mock, not real fault injector")
}
ctx := context.Background()
injector, err := NewNotificationInjector()
if err != nil {
t.Fatalf("[ERROR] Failed to create notification injector: %v", err)
}
// Start the injector - skip if proxy not available
if err := injector.Start(); err != nil {
t.Skipf("Skipping test - proxy not available: %v", err)
}
defer injector.Stop()
// Get test mode configuration
testMode := injector.GetTestModeConfig()
t.Logf("Using %s mode", testMode.Mode)
var reloadCount atomic.Int32
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: injector.GetClusterAddrs(),
Protocol: 3,
})
defer client.Close()
tracker := NewTrackingNotificationsHook()
setupNotificationHook(client, tracker)
client.OnNewNode(func(nodeClient *redis.Client) {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.SetClusterStateReloadCallback(func(ctx context.Context, hostPort string, slotRanges []string) {
reloadCount.Add(1)
})
}
})
if err := client.Ping(ctx).Err(); err != nil {
t.Fatalf("[ERROR] Failed to connect: %v", err)
}
// Perform operations
for i := 0; i < 20; i++ {
client.Set(ctx, fmt.Sprintf("key%d", i), "value", 0)
}
// Wait for connections to establish (mode-aware)
time.Sleep(testMode.ConnectionEstablishDelay)
// Simulate a multi-step migration scenario
t.Log("Step 1: Injecting SMIGRATING for slots 0-5000...")
if err := injector.InjectSMIGRATING(ctx, 10001, "0-5000"); err != nil {
t.Fatalf("[ERROR] Failed to inject SMIGRATING: %v", err)
}
// Wait for notification processing (mode-aware)
time.Sleep(testMode.NotificationDelay)
// Verify operations work during migration
for i := 0; i < 5; i++ {
if err := client.Set(ctx, fmt.Sprintf("migration-key%d", i), "value", 0).Err(); err != nil {
t.Logf("Warning: Operation failed during migration: %v", err)
}
}
if testMode.IsProxyMock() {
// Only inject SMIGRATED with mock injector
t.Log("Step 2: Injecting SMIGRATED for completed migration...")
addrs := injector.GetClusterAddrs()
hostPort := addrs[0]
if err := injector.InjectSMIGRATED(ctx, 10002, hostPort, "0-5000"); err != nil {
t.Fatalf("[ERROR] Failed to inject SMIGRATED: %v", err)
}
// Wait for notification processing (mode-aware)
time.Sleep(testMode.NotificationDelay)
}
// Verify operations still work
for i := 0; i < 5; i++ {
if err := client.Set(ctx, fmt.Sprintf("post-migration-key%d", i), "value", 0).Err(); err != nil {
t.Errorf("[ERROR] Operations failed after migration: %v", err)
}
}
// Print final analysis
analysis := tracker.GetAnalysis()
analysis.Print(t)
t.Logf("✓ Complex scenario test passed")
t.Logf(" - SMIGRATING notifications: %d", analysis.MigratingCount)
t.Logf(" - SMIGRATED notifications: %d", analysis.MigratedCount)
t.Logf(" - Cluster state reloads: %d", reloadCount.Load())
}
maintnotifications/e2e/scripts/run-e2e-tests.sh
+95 -22
View File
@@ -12,7 +12,7 @@ E2E_DIR="${REPO_ROOT}/maintnotifications/e2e"
# Configuration
FAULT_INJECTOR_URL="http://127.0.0.1:20324"
CONFIG_PATH="${REPO_ROOT}/maintnotifications/e2e/infra/cae-client-testing/endpoints.json"
CONFIG_PATH="${REPO_ROOT}/maintnotifications/e2e/infra/cae-client-testing/config/endpoints.json"
# Colors for output
RED='\033[0;31m'
@@ -47,37 +47,50 @@ Usage: $0 [OPTIONS]
OPTIONS:
-h, --help Show this help message
-v, --verbose Enable verbose test output
-t, --timeout DURATION Test timeout (default: 30m)
-v, --verbose Enable verbose test output (human-readable)
-d, --debug Enable debug logging (shows detailed fault injector responses)
-t, --timeout DURATION Test timeout (default: 90m)
-r, --run PATTERN Run only tests matching pattern
--dry-run Show what would be executed without running
--list List available tests
--config PATH Override config path (default: infra/cae-client-testing/endpoints.json)
--fault-injector URL Override fault injector URL (default: http://127.0.0.1:20324)
--json Enable JSON output (default)
--no-json Disable JSON output, use verbose human-readable format
--cluster Run only cluster tests (tests with 'Cluster' in name)
--single Run only non-cluster tests (exclude tests with 'Cluster' in name)
--dry-run Show what would be executed without running
--list List available tests
--config PATH Override config path (default: infra/cae-client-testing/endpoints.json)
--fault-injector URL Override fault injector URL (default: http://127.0.0.1:20324)
EXAMPLES:
$0 # Run all E2E tests
$0 -v # Run with verbose output
$0 -r TestPushNotifications # Run only push notification tests
$0 -t 45m # Run with 45 minute timeout
$0 --dry-run # Show what would be executed
$0 --list # List available tests
$0 # Run all E2E tests with JSON output
$0 --no-json # Run all tests with verbose human-readable output
$0 --debug # Run with debug logging enabled
$0 --cluster # Run only cluster tests (OSS Cluster API)
$0 --single # Run only non-cluster tests
$0 -r TestPushNotifications # Run only tests matching pattern
$0 -t 45m # Run with 45 minute timeout
$0 --dry-run # Show what would be executed
$0 --list # List available tests
ENVIRONMENT:
The script automatically sets up the required environment variables:
- REDIS_ENDPOINTS_CONFIG_PATH: Path to Redis endpoints configuration
- FAULT_INJECTION_API_URL: URL of the fault injector server
- E2E_SCENARIO_TESTS: Enables scenario tests
- E2E_DEBUG: Enables debug logging (set by --debug flag)
EOF
}
# Parse command line arguments
VERBOSE=""
TIMEOUT="30m"
TIMEOUT="90m"
RUN_PATTERN=""
SKIP_PATTERN=""
DRY_RUN=false
LIST_TESTS=false
JSON_OUTPUT=true
CLUSTER_ONLY=false
SINGLE_ONLY=false
DEBUG_MODE=false
while [[ $# -gt 0 ]]; do
case $1 in
@@ -86,7 +99,12 @@ while [[ $# -gt 0 ]]; do
exit 0
;;
-v|--verbose)
VERBOSE="-v"
# Verbose is now the same as --no-json for backward compatibility
JSON_OUTPUT=false
shift
;;
-d|--debug)
DEBUG_MODE=true
shift
;;
-t|--timeout)
@@ -97,6 +115,22 @@ while [[ $# -gt 0 ]]; do
RUN_PATTERN="$2"
shift 2
;;
--json)
JSON_OUTPUT=true
shift
;;
--no-json)
JSON_OUTPUT=false
shift
;;
--cluster)
CLUSTER_ONLY=true
shift
;;
--single)
SINGLE_ONLY=true
shift
;;
--dry-run)
DRY_RUN=true
shift
@@ -121,6 +155,26 @@ while [[ $# -gt 0 ]]; do
esac
done
# Validate mutually exclusive options
if [[ "$CLUSTER_ONLY" == true && "$SINGLE_ONLY" == true ]]; then
log_error "Cannot use --cluster and --single together"
exit 1
fi
# Set run pattern based on cluster/single options
if [[ "$CLUSTER_ONLY" == true ]]; then
if [[ -n "$RUN_PATTERN" ]]; then
log_warning "Overriding -r pattern with --cluster"
fi
RUN_PATTERN="Cluster"
elif [[ "$SINGLE_ONLY" == true ]]; then
if [[ -n "$RUN_PATTERN" ]]; then
log_warning "Overriding -r pattern with --single"
fi
# Use -skip to exclude Cluster tests (requires Go 1.21+)
SKIP_PATTERN="Cluster"
fi
# Validate configuration file exists
if [[ ! -f "$CONFIG_PATH" ]]; then
log_error "Configuration file not found: $CONFIG_PATH"
@@ -132,22 +186,33 @@ fi
export REDIS_ENDPOINTS_CONFIG_PATH="$CONFIG_PATH"
export FAULT_INJECTION_API_URL="$FAULT_INJECTOR_URL"
export E2E_SCENARIO_TESTS="true"
if [[ "$DEBUG_MODE" == true ]]; then
export E2E_DEBUG="true"
fi
# Build test command
TEST_CMD="go test -json -tags=e2e"
TEST_CMD="go test -tags=e2e"
# Add JSON or verbose output
if [[ "$JSON_OUTPUT" == true ]]; then
TEST_CMD="$TEST_CMD -json"
else
TEST_CMD="$TEST_CMD -v"
fi
if [[ -n "$TIMEOUT" ]]; then
TEST_CMD="$TEST_CMD -timeout=$TIMEOUT"
fi
# Note: -v flag is not compatible with -json output format
# The -json format already provides verbose test information
if [[ -n "$RUN_PATTERN" ]]; then
TEST_CMD="$TEST_CMD -run $RUN_PATTERN"
TEST_CMD="$TEST_CMD -run '$RUN_PATTERN'"
fi
TEST_CMD="$TEST_CMD ./maintnotifications/e2e/ "
if [[ -n "$SKIP_PATTERN" ]]; then
TEST_CMD="$TEST_CMD -skip '$SKIP_PATTERN'"
fi
TEST_CMD="$TEST_CMD ./maintnotifications/e2e/"
# List tests if requested
if [[ "$LIST_TESTS" == true ]]; then
@@ -164,9 +229,14 @@ echo " E2E Directory: $E2E_DIR" >&2
echo " Config Path: $CONFIG_PATH" >&2
echo " Fault Injector URL: $FAULT_INJECTOR_URL" >&2
echo " Test Timeout: $TIMEOUT" >&2
echo " JSON Output: $JSON_OUTPUT" >&2
echo " Debug Mode: $DEBUG_MODE" >&2
if [[ -n "$RUN_PATTERN" ]]; then
echo " Test Pattern: $RUN_PATTERN" >&2
fi
if [[ -n "$SKIP_PATTERN" ]]; then
echo " Skip Pattern: $SKIP_PATTERN" >&2
fi
echo "" >&2
# Validate fault injector connectivity
@@ -189,6 +259,9 @@ if [[ "$DRY_RUN" == true ]]; then
echo " export REDIS_ENDPOINTS_CONFIG_PATH=\"$CONFIG_PATH\"" >&2
echo " export FAULT_INJECTION_API_URL=\"$FAULT_INJECTOR_URL\"" >&2
echo " export E2E_SCENARIO_TESTS=\"true\"" >&2
if [[ "$DEBUG_MODE" == true ]]; then
echo " export E2E_DEBUG=\"true\"" >&2
fi
echo " $TEST_CMD" >&2
exit 0
fi
maintnotifications/e2e/slot_keys_test.go
+1691
View File
File diff suppressed because it is too large Load Diff
maintnotifications/e2e/test_mode_test.go
+114
View File
@@ -0,0 +1,114 @@
package e2e
import (
"os"
"time"
)
// TestMode represents the type of test environment
type TestMode int
const (
// TestModeProxyMock uses the local proxy mock for fast testing
TestModeProxyMock TestMode = iota
// TestModeRealFaultInjector uses the real fault injector with actual Redis Enterprise
TestModeRealFaultInjector
)
// TestModeConfig holds configuration for a specific test mode
type TestModeConfig struct {
Mode TestMode
// Timing configuration
NotificationDelay time.Duration // How long to wait after injecting a notification
ActionWaitTimeout time.Duration // How long to wait for fault injector actions
ActionPollInterval time.Duration // How often to poll for action status
DatabaseReadyDelay time.Duration // How long to wait for database to be ready
ConnectionEstablishDelay time.Duration // How long to wait for connections to establish
// Test behavior configuration
MaxClients int // Maximum number of clients to create (proxy mock should use 1)
SkipMultiClientTests bool // Whether to skip tests that require multiple clients
}
// GetTestMode detects the current test mode based on environment variables
func GetTestMode() TestMode {
// If REDIS_ENDPOINTS_CONFIG_PATH is set, we're using real fault injector
if os.Getenv("REDIS_ENDPOINTS_CONFIG_PATH") != "" {
return TestModeRealFaultInjector
}
// If FAULT_INJECTOR_URL is set, we're using real fault injector
if os.Getenv("FAULT_INJECTOR_URL") != "" {
return TestModeRealFaultInjector
}
// Otherwise, we're using proxy mock
return TestModeProxyMock
}
// GetTestModeConfig returns the configuration for the current test mode
func GetTestModeConfig() *TestModeConfig {
mode := GetTestMode()
switch mode {
case TestModeProxyMock:
return &TestModeConfig{
Mode: TestModeProxyMock,
NotificationDelay: 1 * time.Second,
ActionWaitTimeout: 10 * time.Second,
ActionPollInterval: 500 * time.Millisecond,
DatabaseReadyDelay: 1 * time.Second,
ConnectionEstablishDelay: 500 * time.Millisecond,
MaxClients: 1,
SkipMultiClientTests: true,
}
case TestModeRealFaultInjector:
return &TestModeConfig{
Mode: TestModeRealFaultInjector,
NotificationDelay: 30 * time.Second,
ActionWaitTimeout: 5 * time.Minute, // Real fault injector can take up to 5 minutes
ActionPollInterval: 500 * time.Millisecond,
DatabaseReadyDelay: 10 * time.Second,
ConnectionEstablishDelay: 2 * time.Second,
MaxClients: 3,
SkipMultiClientTests: false,
}
default:
// Default to proxy mock for safety
return &TestModeConfig{
Mode: TestModeProxyMock,
NotificationDelay: 1 * time.Second,
ActionWaitTimeout: 10 * time.Second,
ActionPollInterval: 500 * time.Millisecond,
DatabaseReadyDelay: 1 * time.Second,
ConnectionEstablishDelay: 500 * time.Millisecond,
MaxClients: 1,
SkipMultiClientTests: true,
}
}
}
// IsProxyMock returns true if running in proxy mock mode
func (c *TestModeConfig) IsProxyMock() bool {
return c.Mode == TestModeProxyMock
}
// IsRealFaultInjector returns true if running with real fault injector
func (c *TestModeConfig) IsRealFaultInjector() bool {
return c.Mode == TestModeRealFaultInjector
}
// String returns a human-readable name for the test mode
func (m TestMode) String() string {
switch m {
case TestModeProxyMock:
return "ProxyMock"
case TestModeRealFaultInjector:
return "RealFaultInjector"
default:
return "Unknown"
}
}
maintnotifications/manager.go
+47 -5
View File
@@ -18,11 +18,13 @@ import (
// Push notification type constants for maintenance
const (
NotificationMoving = "MOVING"
NotificationMigrating = "MIGRATING"
NotificationMigrated = "MIGRATED"
NotificationFailingOver = "FAILING_OVER"
NotificationFailedOver = "FAILED_OVER"
NotificationMoving = "MOVING" // Per-connection handoff notification
NotificationMigrating = "MIGRATING" // Per-connection migration start notification - relaxes timeouts
NotificationMigrated = "MIGRATED" // Per-connection migration complete notification - clears relaxed timeouts
NotificationFailingOver = "FAILING_OVER" // Per-connection failover start notification - relaxes timeouts
NotificationFailedOver = "FAILED_OVER" // Per-connection failover complete notification - clears relaxed timeouts
NotificationSMigrating = "SMIGRATING" // Cluster slot migrating notification - relaxes timeouts
NotificationSMigrated = "SMIGRATED" // Cluster slot migrated notification - unrelaxes timeouts and triggers cluster state reload
)
// maintenanceNotificationTypes contains all notification types that maintenance handles
@@ -32,6 +34,8 @@ var maintenanceNotificationTypes = []string{
NotificationMigrated,
NotificationFailingOver,
NotificationFailedOver,
NotificationSMigrating,
NotificationSMigrated,
}
// NotificationHook is called before and after notification processing
@@ -65,6 +69,10 @@ type Manager struct {
// MOVING operation tracking - using sync.Map for better concurrent performance
activeMovingOps sync.Map // map[MovingOperationKey]*MovingOperation
// SMIGRATED notification deduplication - tracks processed SeqIDs
// Multiple connections may receive the same SMIGRATED notification
processedSMigratedSeqIDs sync.Map // map[int64]bool
// Atomic state tracking - no locks needed for state queries
activeOperationCount atomic.Int64 // Number of active operations
closed atomic.Bool // Manager closed state
@@ -73,6 +81,9 @@ type Manager struct {
hooks []NotificationHook
hooksMu sync.RWMutex // Protects hooks slice
poolHooksRef *PoolHook
// Cluster state reload callback for SMIGRATED notifications
clusterStateReloadCallback ClusterStateReloadCallback
}
// MovingOperation tracks an active MOVING operation.
@@ -83,6 +94,14 @@ type MovingOperation struct {
Deadline time.Time
}
// ClusterStateReloadCallback is a callback function that triggers cluster state reload.
// This is used by node clients to notify their parent ClusterClient about SMIGRATED notifications.
// The hostPort parameter indicates the destination node (e.g., "127.0.0.1:6379").
// The slotRanges parameter contains the migrated slots (e.g., ["1234", "5000-6000"]).
// Currently, implementations typically reload the entire cluster state, but in the future
// this could be optimized to reload only the specific slots.
type ClusterStateReloadCallback func(ctx context.Context, hostPort string, slotRanges []string)
// NewManager creates a new simplified manager.
func NewManager(client interfaces.ClientInterface, pool pool.Pooler, config *Config) (*Manager, error) {
if client == nil {
@@ -223,6 +242,15 @@ func (hm *Manager) GetActiveOperationCount() int64 {
return hm.activeOperationCount.Load()
}
// MarkSMigratedSeqIDProcessed attempts to mark a SMIGRATED SeqID as processed.
// Returns true if this is the first time processing this SeqID (should process),
// false if it was already processed (should skip).
// This prevents duplicate processing when multiple connections receive the same notification.
func (hm *Manager) MarkSMigratedSeqIDProcessed(seqID int64) bool {
_, alreadyProcessed := hm.processedSMigratedSeqIDs.LoadOrStore(seqID, true)
return !alreadyProcessed // Return true if NOT already processed
}
// Close closes the manager.
func (hm *Manager) Close() error {
// Use atomic operation for thread-safe close check
@@ -318,3 +346,17 @@ func (hm *Manager) AddNotificationHook(notificationHook NotificationHook) {
defer hm.hooksMu.Unlock()
hm.hooks = append(hm.hooks, notificationHook)
}
// SetClusterStateReloadCallback sets the callback function that will be called when a SMIGRATED notification is received.
// This allows node clients to notify their parent ClusterClient to reload cluster state.
func (hm *Manager) SetClusterStateReloadCallback(callback ClusterStateReloadCallback) {
hm.clusterStateReloadCallback = callback
}
// TriggerClusterStateReload calls the cluster state reload callback if it's set.
// This is called when a SMIGRATED notification is received.
func (hm *Manager) TriggerClusterStateReload(ctx context.Context, hostPort string, slotRanges []string) {
if hm.clusterStateReloadCallback != nil {
hm.clusterStateReloadCallback(ctx, hostPort, slotRanges)
}
}
maintnotifications/manager_test.go
+6
View File
@@ -52,6 +52,10 @@ func (mo *MockOptions) GetAddr() string {
return "localhost:6379"
}
func (mo *MockOptions) GetNodeAddress() string {
return "localhost:6379"
}
func (mo *MockOptions) IsTLSEnabled() bool {
return false
}
@@ -217,6 +221,8 @@ func TestManagerRefactoring(t *testing.T) {
NotificationMigrated,
NotificationFailingOver,
NotificationFailedOver,
NotificationSMigrating,
NotificationSMigrated,
}
if len(maintenanceNotificationTypes) != len(expectedTypes) {
maintnotifications/pool_hook_test.go
+18 -2
View File
@@ -700,9 +700,25 @@ func TestConnectionHook(t *testing.T) {
t.Errorf("Connection should be pooled after handoff (shouldPool=%v, shouldRemove=%v)", shouldPool, shouldRemove)
}
// Wait for handoff to complete
time.Sleep(50 * time.Millisecond)
// Wait for handoff to complete with polling instead of fixed sleep
// This avoids flakiness on slow CI runners where 50ms may not be enough
maxWait := 500 * time.Millisecond
pollInterval := 10 * time.Millisecond
deadline := time.Now().Add(maxWait)
handoffCompleted := false
for time.Now().Before(deadline) {
if conn.IsUsable() && !processor.IsHandoffPending(conn) {
handoffCompleted = true
break
}
time.Sleep(pollInterval)
}
if !handoffCompleted {
t.Fatalf("Handoff did not complete within %v (IsUsable=%v, IsHandoffPending=%v)",
maxWait, conn.IsUsable(), processor.IsHandoffPending(conn))
}
// After handoff completion, connection should be usable again
if !conn.IsUsable() {
t.Error("Connection should be usable after handoff completion")
maintnotifications/push_notification_handler.go
+232 -9
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/redis/go-redis/v9/internal"
@@ -49,6 +50,10 @@ func (snh *NotificationHandler) HandlePushNotification(ctx context.Context, hand
err = snh.handleFailingOver(ctx, handlerCtx, modifiedNotification)
case NotificationFailedOver:
err = snh.handleFailedOver(ctx, handlerCtx, modifiedNotification)
case NotificationSMigrating:
err = snh.handleSMigrating(ctx, handlerCtx, modifiedNotification)
case NotificationSMigrated:
err = snh.handleSMigrated(ctx, handlerCtx, modifiedNotification)
default:
// Ignore other notification types (e.g., pub/sub messages)
err = nil
@@ -61,7 +66,9 @@ func (snh *NotificationHandler) HandlePushNotification(ctx context.Context, hand
}
// handleMoving processes MOVING notifications.
// ["MOVING", seqNum, timeS, endpoint] - per-connection handoff
// MOVING indicates that a connection should be handed off to a new endpoint.
// This is a per-connection notification that triggers connection handoff.
// Expected format: ["MOVING", seqNum, timeS, endpoint]
func (snh *NotificationHandler) handleMoving(ctx context.Context, handlerCtx push.NotificationHandlerContext, notification []interface{}) error {
if len(notification) < 3 {
internal.Logger.Printf(ctx, logs.InvalidNotification("MOVING", notification))
@@ -140,7 +147,28 @@ func (snh *NotificationHandler) handleMoving(ctx context.Context, handlerCtx pus
if err := snh.markConnForHandoff(poolConn, newEndpoint, seqID, deadline); err != nil {
// Log error but don't fail the goroutine - use background context since original may be cancelled
internal.Logger.Printf(context.Background(), logs.FailedToMarkForHandoff(poolConn.GetID(), err))
return
}
// Queue the handoff immediately if the connection is idle in the pool.
// If the connection is in use (StateInUse), it will be queued when returned to the pool via OnPut.
// This handles the case where the connection is idle and might never be retrieved again.
if poolConn.GetStateMachine().GetState() == pool.StateIdle {
if snh.manager.poolHooksRef != nil && snh.manager.poolHooksRef.workerManager != nil {
if err := snh.manager.poolHooksRef.workerManager.queueHandoff(poolConn); err != nil {
internal.Logger.Printf(context.Background(), logs.FailedToQueueHandoff(poolConn.GetID(), err))
} else {
// Mark the connection as queued for handoff to prevent it from being retrieved
// This transitions the connection to StateUnusable
if err := poolConn.MarkQueuedForHandoff(); err != nil {
internal.Logger.Printf(context.Background(), logs.FailedToMarkForHandoff(poolConn.GetID(), err))
} else {
internal.Logger.Printf(context.Background(), logs.MarkedForHandoff(poolConn.GetID()))
}
}
}
}
// If connection is StateInUse, the handoff will be queued when it's returned to the pool
})
return nil
}
@@ -167,9 +195,10 @@ func (snh *NotificationHandler) markConnForHandoff(conn *pool.Conn, newEndpoint
}
// handleMigrating processes MIGRATING notifications.
// MIGRATING indicates that a connection migration is starting.
// This is a per-connection notification that applies relaxed timeouts.
// Expected format: ["MIGRATING", ...]
func (snh *NotificationHandler) handleMigrating(ctx context.Context, handlerCtx push.NotificationHandlerContext, notification []interface{}) error {
// MIGRATING notifications indicate that a connection is about to be migrated
// Apply relaxed timeouts to the specific connection that received this notification
if len(notification) < 2 {
internal.Logger.Printf(ctx, logs.InvalidNotification("MIGRATING", notification))
return ErrInvalidNotification
@@ -195,9 +224,10 @@ func (snh *NotificationHandler) handleMigrating(ctx context.Context, handlerCtx
}
// handleMigrated processes MIGRATED notifications.
// MIGRATED indicates that a connection migration has completed.
// This is a per-connection notification that clears relaxed timeouts.
// Expected format: ["MIGRATED", ...]
func (snh *NotificationHandler) handleMigrated(ctx context.Context, handlerCtx push.NotificationHandlerContext, notification []interface{}) error {
// MIGRATED notifications indicate that a connection migration has completed
// Restore normal timeouts for the specific connection that received this notification
if len(notification) < 2 {
internal.Logger.Printf(ctx, logs.InvalidNotification("MIGRATED", notification))
return ErrInvalidNotification
@@ -224,9 +254,10 @@ func (snh *NotificationHandler) handleMigrated(ctx context.Context, handlerCtx p
}
// handleFailingOver processes FAILING_OVER notifications.
// FAILING_OVER indicates that a failover is starting.
// This is a per-connection notification that applies relaxed timeouts.
// Expected format: ["FAILING_OVER", ...]
func (snh *NotificationHandler) handleFailingOver(ctx context.Context, handlerCtx push.NotificationHandlerContext, notification []interface{}) error {
// FAILING_OVER notifications indicate that a connection is about to failover
// Apply relaxed timeouts to the specific connection that received this notification
if len(notification) < 2 {
internal.Logger.Printf(ctx, logs.InvalidNotification("FAILING_OVER", notification))
return ErrInvalidNotification
@@ -253,9 +284,10 @@ func (snh *NotificationHandler) handleFailingOver(ctx context.Context, handlerCt
}
// handleFailedOver processes FAILED_OVER notifications.
// FAILED_OVER indicates that a failover has completed.
// This is a per-connection notification that clears relaxed timeouts.
// Expected format: ["FAILED_OVER", ...]
func (snh *NotificationHandler) handleFailedOver(ctx context.Context, handlerCtx push.NotificationHandlerContext, notification []interface{}) error {
// FAILED_OVER notifications indicate that a connection failover has completed
// Restore normal timeouts for the specific connection that received this notification
if len(notification) < 2 {
internal.Logger.Printf(ctx, logs.InvalidNotification("FAILED_OVER", notification))
return ErrInvalidNotification
@@ -280,3 +312,194 @@ func (snh *NotificationHandler) handleFailedOver(ctx context.Context, handlerCtx
conn.ClearRelaxedTimeout()
return nil
}
// handleSMigrating processes SMIGRATING notifications.
// SMIGRATING indicates that a cluster slot is in the process of migrating to a different node.
// This is a per-connection notification that applies relaxed timeouts during slot migration.
// Expected format: ["SMIGRATING", SeqID, slot/range1-range2, ...]
func (snh *NotificationHandler) handleSMigrating(ctx context.Context, handlerCtx push.NotificationHandlerContext, notification []interface{}) error {
if len(notification) < 3 {
internal.Logger.Printf(ctx, logs.InvalidNotification("SMIGRATING", notification))
return ErrInvalidNotification
}
// Validate SeqID (position 1)
if _, ok := notification[1].(int64); !ok {
internal.Logger.Printf(ctx, logs.InvalidSeqIDInSMigratingNotification(notification[1]))
return ErrInvalidNotification
}
if handlerCtx.Conn == nil {
internal.Logger.Printf(ctx, logs.NoConnectionInHandlerContext("SMIGRATING"))
return ErrInvalidNotification
}
conn, ok := handlerCtx.Conn.(*pool.Conn)
if !ok {
internal.Logger.Printf(ctx, logs.InvalidConnectionTypeInHandlerContext("SMIGRATING", handlerCtx.Conn, handlerCtx))
return ErrInvalidNotification
}
// Apply relaxed timeout to this specific connection
if internal.LogLevel.InfoOrAbove() {
internal.Logger.Printf(ctx, logs.RelaxedTimeoutDueToNotification(conn.GetID(), "SMIGRATING", snh.manager.config.RelaxedTimeout))
}
conn.SetRelaxedTimeout(snh.manager.config.RelaxedTimeout, snh.manager.config.RelaxedTimeout)
return nil
}
// handleSMigrated processes SMIGRATED notifications.
// SMIGRATED indicates that a cluster slot has finished migrating to a different node.
// This is a cluster-level notification that triggers cluster state reload.
//
// Expected RESP3 format:
//
// >3
// +SMIGRATED
// :SeqID
// *<num_entries> <- array of triplet arrays
// *3 <- each triplet is a 3-element array
// +<source> <- node from which slots are migrating FROM
// +<destination> <- node to which slots are migrating TO
// +<slots> <- comma-separated slots and/or ranges (e.g., "123,789-1000")
//
// A source and target endpoint may appear in multiple triplets.
// The notification is only processed if the connection's NodeAddress matches one of the source endpoints.
//
// Note: Multiple connections may receive the same notification, so we deduplicate by SeqID before triggering reload.
// but we still process the notification on each connection to clear the relaxed timeout.
// In the case when the connection is from MOVED/ASK, the connection's original endpoint is not set,
// so we will not be able to match the source endpoint. In such case, we will trigger the reload callback with the first target endpoint.
func (snh *NotificationHandler) handleSMigrated(ctx context.Context, handlerCtx push.NotificationHandlerContext, notification []interface{}) error {
// Expected: ["SMIGRATED", SeqID, [[source, target, slots], ...]]
// Minimum 3 elements: SMIGRATED, SeqID, and the array of triplets
if len(notification) < 3 {
internal.Logger.Printf(ctx, logs.InvalidNotification("SMIGRATED", notification))
return ErrInvalidNotification
}
// Extract SeqID (position 1)
seqID, ok := notification[1].(int64)
if !ok {
internal.Logger.Printf(ctx, logs.InvalidSeqIDInSMigratedNotification(notification[1]))
return ErrInvalidNotification
}
// Extract the array of triplets (position 2)
triplets, ok := notification[2].([]interface{})
if !ok {
internal.Logger.Printf(ctx, logs.InvalidNotification("SMIGRATED (triplets array)", notification[2]))
return ErrInvalidNotification
}
if len(triplets) == 0 {
internal.Logger.Printf(ctx, logs.InvalidNotification("SMIGRATED (empty triplets)", notification))
return ErrInvalidNotification
}
// Get the connection's endpoints to check if this notification is relevant
// We check against both nodeAddress (from CLUSTER SLOTS) and addr (after resolution)
// since we cannot be certain which format the notification source will use
var connectionNodeAddress string
var connectionAddr string
if snh.manager.options != nil {
connectionNodeAddress = snh.manager.options.GetNodeAddress()
connectionAddr = snh.manager.options.GetAddr()
}
// Helper function to check if source matches either of our endpoints
// notification source can be either the node address or the addr after resolution
sourceMatchesConnection := func(source string) bool {
if source == connectionNodeAddress {
return true
}
if source == connectionAddr {
return true
}
return false
}
// Parse triplets and check if any source matches our connection's endpoints
var matchingTriplets []struct {
source string
target string
slots string
}
var allSlotRanges []string
for _, tripletInterface := range triplets {
// Each triplet should be a 3-element array: [source, target, slots]
triplet, ok := tripletInterface.([]interface{})
if !ok || len(triplet) != 3 {
internal.Logger.Printf(ctx, logs.InvalidNotification("SMIGRATED (triplet format)", tripletInterface))
continue
}
// Extract source endpoint
source, ok := triplet[0].(string)
if !ok {
internal.Logger.Printf(ctx, logs.InvalidNotification("SMIGRATED (source)", triplet[0]))
continue
}
// Extract target endpoint
target, ok := triplet[1].(string)
if !ok {
internal.Logger.Printf(ctx, logs.InvalidNotification("SMIGRATED (target)", triplet[1]))
continue
}
// Extract slots
slots, ok := triplet[2].(string)
if !ok {
internal.Logger.Printf(ctx, logs.InvalidNotification("SMIGRATED (slots)", triplet[2]))
continue
}
// Check if this triplet's source matches our connection's endpoints
if sourceMatchesConnection(source) {
matchingTriplets = append(matchingTriplets, struct {
source string
target string
slots string
}{source, target, slots})
slotRanges := strings.Split(slots, ",")
allSlotRanges = append(allSlotRanges, slotRanges...)
}
}
var connID uint64
// Reset relaxed timeout for this specific connection
if handlerCtx.Conn != nil {
conn, ok := handlerCtx.Conn.(*pool.Conn)
if ok {
if internal.LogLevel.InfoOrAbove() {
connID = conn.GetID()
internal.Logger.Printf(ctx, logs.UnrelaxedTimeout(connID))
}
conn.ClearRelaxedTimeout()
}
}
// If no matching triplets, this notification is not relevant to this connection
if len(matchingTriplets) == 0 {
return nil
}
// Deduplicate by SeqID - multiple connections may receive the same notification
// Only trigger cluster state reload once per seqID
if snh.manager.MarkSMigratedSeqIDProcessed(seqID) {
// Use the first matching triplet
target := matchingTriplets[0].target
slotsForLog := allSlotRanges
if internal.LogLevel.InfoOrAbove() {
internal.Logger.Printf(ctx, logs.TriggeringClusterStateReload(seqID, target, slotsForLog))
}
// Trigger cluster state reload via callback
snh.manager.TriggerClusterStateReload(ctx, target, slotsForLog)
}
return nil
}
maintnotifications/push_notification_handler_test.go
+236
View File
@@ -0,0 +1,236 @@
package maintnotifications
import (
"context"
"net"
"testing"
"time"
"github.com/redis/go-redis/v9/internal/interfaces"
"github.com/redis/go-redis/v9/push"
)
// testOptions implements interfaces.OptionsInterface for testing SMIGRATED
type testOptions struct {
nodeAddress string
}
func (to *testOptions) GetReadTimeout() time.Duration { return 5 * time.Second }
func (to *testOptions) GetWriteTimeout() time.Duration { return 5 * time.Second }
func (to *testOptions) GetNetwork() string { return "tcp" }
func (to *testOptions) GetAddr() string { return "localhost:6379" }
func (to *testOptions) GetNodeAddress() string { return to.nodeAddress }
func (to *testOptions) IsTLSEnabled() bool { return false }
func (to *testOptions) GetProtocol() int { return 3 }
func (to *testOptions) GetPoolSize() int { return 10 }
func (to *testOptions) NewDialer() func(context.Context) (net.Conn, error) {
return func(ctx context.Context) (net.Conn, error) { return nil, nil }
}
func createTestManager(nodeAddress string) *Manager {
config := DefaultConfig()
var opts interfaces.OptionsInterface
if nodeAddress != "" {
opts = &testOptions{nodeAddress: nodeAddress}
}
return &Manager{
config: config,
options: opts,
}
}
// TestHandleSMigrated_CorrectFormat tests that handleSMigrated correctly parses the nested array format
func TestHandleSMigrated_CorrectFormat(t *testing.T) {
// Create a manager with matching original endpoint
manager := createTestManager("127.0.0.1:6379")
handler := &NotificationHandler{
manager: manager,
}
// Create notification in the correct nested array format:
// ["SMIGRATED", SeqID, [[source, target, slots], [source, target, slots], ...]]
notification := []interface{}{
"SMIGRATED",
int64(12346),
[]interface{}{
[]interface{}{"127.0.0.1:6379", "127.0.0.1:6380", "123,456,789-1000"},
[]interface{}{"127.0.0.1:6379", "127.0.0.1:6381", "124,457,300-500"},
},
}
ctx := context.Background()
handlerCtx := push.NotificationHandlerContext{
Conn: nil, // No connection needed for this test
}
// This should not return an error
err := handler.handleSMigrated(ctx, handlerCtx, notification)
if err != nil {
t.Errorf("handleSMigrated failed with correct format: %v", err)
}
}
// TestHandleSMigrated_SingleTriplet tests parsing with a single triplet
func TestHandleSMigrated_SingleTriplet(t *testing.T) {
manager := createTestManager("127.0.0.1:6379")
handler := &NotificationHandler{
manager: manager,
}
// Single triplet in nested array format
notification := []interface{}{
"SMIGRATED",
int64(100),
[]interface{}{
[]interface{}{"127.0.0.1:6379", "127.0.0.1:6380", "1000,2000-3000"},
},
}
ctx := context.Background()
handlerCtx := push.NotificationHandlerContext{
Conn: nil,
}
err := handler.handleSMigrated(ctx, handlerCtx, notification)
if err != nil {
t.Errorf("handleSMigrated failed with single triplet: %v", err)
}
}
// TestHandleSMigrated_NoMatchingSource tests that notification is ignored when source doesn't match
func TestHandleSMigrated_NoMatchingSource(t *testing.T) {
// Create a manager with a different original endpoint
manager := createTestManager("127.0.0.1:9999")
handler := &NotificationHandler{
manager: manager,
}
// Notification with source that doesn't match our endpoint
notification := []interface{}{
"SMIGRATED",
int64(200),
[]interface{}{
[]interface{}{"127.0.0.1:6379", "127.0.0.1:6380", "1000,2000-3000"},
},
}
ctx := context.Background()
handlerCtx := push.NotificationHandlerContext{
Conn: nil,
}
// Should not return an error, just silently ignore
err := handler.handleSMigrated(ctx, handlerCtx, notification)
if err != nil {
t.Errorf("handleSMigrated should not error when source doesn't match: %v", err)
}
}
// TestHandleSMigrated_IncompleteTriplet tests that incomplete triplets are skipped
func TestHandleSMigrated_IncompleteTriplet(t *testing.T) {
manager := createTestManager("127.0.0.1:6379")
handler := &NotificationHandler{
manager: manager,
}
// Triplet with only 2 elements (missing slots) - should be skipped
notification := []interface{}{
"SMIGRATED",
int64(300),
[]interface{}{
[]interface{}{"127.0.0.1:6379", "127.0.0.1:6380"}, // Missing slots
},
}
ctx := context.Background()
handlerCtx := push.NotificationHandlerContext{
Conn: nil,
}
// Should not error, just skip the malformed triplet
err := handler.handleSMigrated(ctx, handlerCtx, notification)
if err != nil {
t.Errorf("handleSMigrated should not error with incomplete triplet (just skip it): %v", err)
}
}
// TestHandleSMigrated_TooFewElements tests that notifications with too few elements are rejected
func TestHandleSMigrated_TooFewElements(t *testing.T) {
manager := createTestManager("127.0.0.1:6379")
handler := &NotificationHandler{
manager: manager,
}
// Only SMIGRATED and SeqID, no triplets array
notification := []interface{}{
"SMIGRATED",
int64(400),
}
ctx := context.Background()
handlerCtx := push.NotificationHandlerContext{
Conn: nil,
}
err := handler.handleSMigrated(ctx, handlerCtx, notification)
if err == nil {
t.Error("handleSMigrated should error with too few elements")
}
}
// TestHandleSMigrated_EmptyTripletsArray tests that empty triplets array is rejected
func TestHandleSMigrated_EmptyTripletsArray(t *testing.T) {
manager := createTestManager("127.0.0.1:6379")
handler := &NotificationHandler{
manager: manager,
}
// Empty triplets array
notification := []interface{}{
"SMIGRATED",
int64(500),
[]interface{}{},
}
ctx := context.Background()
handlerCtx := push.NotificationHandlerContext{
Conn: nil,
}
err := handler.handleSMigrated(ctx, handlerCtx, notification)
if err == nil {
t.Error("handleSMigrated should error with empty triplets array")
}
}
// TestHandleSMigrated_InvalidTripletsType tests that non-array triplets is rejected
func TestHandleSMigrated_InvalidTripletsType(t *testing.T) {
manager := createTestManager("127.0.0.1:6379")
handler := &NotificationHandler{
manager: manager,
}
// Triplets is a string instead of array
notification := []interface{}{
"SMIGRATED",
int64(600),
"not an array",
}
ctx := context.Background()
handlerCtx := push.NotificationHandlerContext{
Conn: nil,
}
err := handler.handleSMigrated(ctx, handlerCtx, notification)
if err == nil {
t.Error("handleSMigrated should error with invalid triplets type")
}
}
options.go
+17
View File
@@ -42,6 +42,17 @@ type Options struct {
// Addr is the address formated as host:port
Addr string
// NodeAddress is the address of the Redis node as reported by the server.
// For cluster clients, this is the exact endpoint string returned by CLUSTER SLOTS
// before any resolution or transformation (e.g., loopback replacement).
// For standalone clients, this defaults to Addr.
//
// This is used to match the source endpoint in maintenance notifications
// (e.g. SMIGRATED).
//
// Use Client.NodeAddress() to access this value.
NodeAddress string
// ClientName will execute the `CLIENT SETNAME ClientName` command for each conn.
ClientName string
@@ -295,6 +306,12 @@ func (opt *Options) init() {
opt.Network = "tcp"
}
}
// For standalone clients, default NodeAddress to Addr if not set.
// This ensures maintenance notifications (SMIGRATED, etc.) can match
// the connection's endpoint even for non-cluster clients.
if opt.NodeAddress == "" {
opt.NodeAddress = opt.Addr
}
if opt.Protocol < 2 {
opt.Protocol = 3
}
osscluster.go
+129 -35
View File
@@ -107,6 +107,10 @@ type ClusterOptions struct {
WriteTimeout time.Duration
ContextTimeoutEnabled bool
// MaxConcurrentDials is the maximum number of concurrent connection creation goroutines.
// If <= 0, defaults to PoolSize. If > PoolSize, it will be capped at PoolSize.
MaxConcurrentDials int
PoolFIFO bool
PoolSize int // applies per cluster node and not for the whole cluster
PoolTimeout time.Duration
@@ -169,11 +173,16 @@ type ClusterOptions struct {
// cluster upgrade notifications gracefully and manage connection/pool state
// transitions seamlessly. Requires Protocol: 3 (RESP3) for push notifications.
// If nil, maintnotifications upgrades are in "auto" mode and will be enabled if the server supports it.
// The ClusterClient does not directly work with maintnotifications, it is up to the clients in the Nodes map to work with maintnotifications.
// The ClusterClient supports SMIGRATING and SMIGRATED notifications for cluster state management.
// Individual node clients handle other maintenance notifications (MOVING, MIGRATING, etc.).
MaintNotificationsConfig *maintnotifications.Config
// ShardPicker is used to pick a shard when the request_policy is
// ReqDefault and the command has no keys.
ShardPicker routing.ShardPicker
// ClusterStateReloadInterval is the interval for reloading the cluster state.
// Default is 10 seconds.
ClusterStateReloadInterval time.Duration
}
func (opt *ClusterOptions) init() {
@@ -188,9 +197,24 @@ func (opt *ClusterOptions) init() {
opt.ReadOnly = true
}
if opt.DialTimeout == 0 {
opt.DialTimeout = 5 * time.Second
}
if opt.DialerRetries == 0 {
opt.DialerRetries = 5
}
if opt.DialerRetryTimeout == 0 {
opt.DialerRetryTimeout = 100 * time.Millisecond
}
if opt.PoolSize == 0 {
opt.PoolSize = 5 * runtime.GOMAXPROCS(0)
}
if opt.MaxConcurrentDials <= 0 {
opt.MaxConcurrentDials = opt.PoolSize
} else if opt.MaxConcurrentDials > opt.PoolSize {
opt.MaxConcurrentDials = opt.PoolSize
}
if opt.ReadBufferSize == 0 {
opt.ReadBufferSize = proto.DefaultBufferSize
}
@@ -238,6 +262,10 @@ func (opt *ClusterOptions) init() {
if opt.ShardPicker == nil {
opt.ShardPicker = &routing.RoundRobinPicker{}
}
if opt.ClusterStateReloadInterval == 0 {
opt.ClusterStateReloadInterval = 10 * time.Second
}
}
// ParseClusterURL parses a URL into ClusterOptions that can be used to connect to Redis.
@@ -332,10 +360,13 @@ func setupClusterQueryParams(u *url.URL, o *ClusterOptions) (*ClusterOptions, er
o.MinRetryBackoff = q.duration("min_retry_backoff")
o.MaxRetryBackoff = q.duration("max_retry_backoff")
o.DialTimeout = q.duration("dial_timeout")
o.DialerRetries = q.int("dialer_retries")
o.DialerRetryTimeout = q.duration("dialer_retry_timeout")
o.ReadTimeout = q.duration("read_timeout")
o.WriteTimeout = q.duration("write_timeout")
o.PoolFIFO = q.bool("pool_fifo")
o.PoolSize = q.int("pool_size")
o.MaxConcurrentDials = q.int("max_concurrent_dials")
o.MinIdleConns = q.int("min_idle_conns")
o.MaxIdleConns = q.int("max_idle_conns")
o.MaxActiveConns = q.int("max_active_conns")
@@ -394,15 +425,17 @@ func (opt *ClusterOptions) clientOptions() *Options {
MinRetryBackoff: opt.MinRetryBackoff,
MaxRetryBackoff: opt.MaxRetryBackoff,
DialTimeout: opt.DialTimeout,
DialerRetries: opt.DialerRetries,
DialerRetryTimeout: opt.DialerRetryTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
DialTimeout: opt.DialTimeout,
DialerRetries: opt.DialerRetries,
DialerRetryTimeout: opt.DialerRetryTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
ContextTimeoutEnabled: opt.ContextTimeoutEnabled,
PoolFIFO: opt.PoolFIFO,
PoolSize: opt.PoolSize,
MaxConcurrentDials: opt.MaxConcurrentDials,
PoolTimeout: opt.PoolTimeout,
MinIdleConns: opt.MinIdleConns,
MaxIdleConns: opt.MaxIdleConns,
@@ -443,9 +476,10 @@ type clusterNode struct {
lastLatencyMeasurement int64 // atomic
}
func newClusterNode(clOpt *ClusterOptions, addr string) *clusterNode {
func newClusterNodeWithNodeAddress(clOpt *ClusterOptions, addr, nodeAddress string) *clusterNode {
opt := clOpt.clientOptions()
opt.Addr = addr
opt.NodeAddress = nodeAddress
node := clusterNode{
Client: clOpt.NewClient(opt),
}
@@ -673,6 +707,10 @@ func (c *clusterNodes) GC(generation uint32) {
}
func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) {
return c.GetOrCreateWithNodeAddress(addr, "")
}
func (c *clusterNodes) GetOrCreateWithNodeAddress(addr, nodeAddress string) (*clusterNode, error) {
node, err := c.get(addr)
if err != nil {
return nil, err
@@ -693,7 +731,7 @@ func (c *clusterNodes) GetOrCreate(addr string) (*clusterNode, error) {
return node, nil
}
node = newClusterNode(c.opt, addr)
node = newClusterNodeWithNodeAddress(c.opt, addr, nodeAddress)
for _, fn := range c.onNewNode {
fn(node.Client)
}
@@ -790,12 +828,14 @@ func newClusterState(
for _, slot := range slots {
var nodes []*clusterNode
for i, slotNode := range slot.Nodes {
addr := slotNode.Addr
// slotNode.Addr is the node address from CLUSTER SLOTS
nodeAddress := slotNode.Addr
addr := nodeAddress
if !isLoopbackOrigin {
addr = replaceLoopbackHost(addr, originHost)
}
node, err := c.nodes.GetOrCreate(addr)
node, err := c.nodes.GetOrCreateWithNodeAddress(addr, nodeAddress)
if err != nil {
return nil, err
}
@@ -1002,14 +1042,18 @@ func (c *clusterState) slotNodes(slot int) []*clusterNode {
//------------------------------------------------------------------------------
type clusterStateHolder struct {
load func(ctx context.Context) (*clusterState, error)
state atomic.Value
reloading uint32 // atomic
load func(ctx context.Context) (*clusterState, error)
reloadInterval time.Duration
state atomic.Value
reloading uint32 // atomic
reloadPending uint32 // atomic - set to 1 when reload is requested during active reload
}
func newClusterStateHolder(load func(ctx context.Context) (*clusterState, error)) *clusterStateHolder {
func newClusterStateHolder(load func(ctx context.Context) (*clusterState, error), reloadInterval time.Duration) *clusterStateHolder {
return &clusterStateHolder{
load: load,
load: load,
reloadInterval: reloadInterval,
}
}
@@ -1023,17 +1067,37 @@ func (c *clusterStateHolder) Reload(ctx context.Context) (*clusterState, error)
}
func (c *clusterStateHolder) LazyReload() {
// If already reloading, mark that another reload is pending
if !atomic.CompareAndSwapUint32(&c.reloading, 0, 1) {
atomic.StoreUint32(&c.reloadPending, 1)
return
}
go func() {
defer atomic.StoreUint32(&c.reloading, 0)
_, err := c.Reload(context.Background())
if err != nil {
return
go func() {
for {
_, err := c.Reload(context.Background())
if err != nil {
atomic.StoreUint32(&c.reloadPending, 0)
atomic.StoreUint32(&c.reloading, 0)
return
}
// Clear pending flag after reload completes, before cooldown
// This captures notifications that arrived during the reload
atomic.StoreUint32(&c.reloadPending, 0)
// Wait cooldown period
time.Sleep(200 * time.Millisecond)
// Check if another reload was requested during cooldown
if atomic.LoadUint32(&c.reloadPending) == 0 {
// No pending reload, we're done
atomic.StoreUint32(&c.reloading, 0)
return
}
// Pending reload requested, loop to reload again
}
time.Sleep(200 * time.Millisecond)
}()
}
@@ -1044,7 +1108,7 @@ func (c *clusterStateHolder) Get(ctx context.Context) (*clusterState, error) {
}
state := v.(*clusterState)
if time.Since(state.createdAt) > 10*time.Second {
if time.Since(state.createdAt) > c.reloadInterval {
c.LazyReload()
}
return state, nil
@@ -1085,7 +1149,7 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient {
c.cmdsInfoCache = newCmdsInfoCache(c.cmdsInfo)
c.state = newClusterStateHolder(c.loadState)
c.state = newClusterStateHolder(c.loadState, opt.ClusterStateReloadInterval)
c.SetCommandInfoResolver(NewDefaultCommandPolicyResolver())
@@ -1097,6 +1161,26 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient {
txPipeline: c.processTxPipeline,
})
// Set up SMIGRATED notification handling for cluster state reload
// When a node client receives a SMIGRATED notification, it should trigger
// cluster state reload on the parent ClusterClient
if opt.MaintNotificationsConfig != nil {
c.nodes.OnNewNode(func(nodeClient *Client) {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.SetClusterStateReloadCallback(func(ctx context.Context, hostPort string, slotRanges []string) {
// Log the migration details for now
if internal.LogLevel.InfoOrAbove() {
internal.Logger.Printf(ctx, "cluster: slots %v migrated to %s, reloading cluster state", slotRanges, hostPort)
}
// Currently we reload the entire cluster state
// In the future, this could be optimized to reload only the specific slots
c.state.LazyReload()
})
}
})
}
return c
}
@@ -1492,14 +1576,20 @@ func (c *ClusterClient) mapCmdsByNode(ctx context.Context, cmdsMap *cmdsMap, cmd
}
slot := c.cmdSlot(cmd, -1)
var node *clusterNode
if slot == -1 {
// Keyless command - use ShardPicker to select a node
node, err = c.cmdNodeWithShardPicker(ctx, cmd.Name(), slot, c.opt.ShardPicker)
// For keyless commands (slot == -1), use ShardPicker if routing policies are enabled
if slot == -1 && !c.opt.DisableRoutingPolicies && c.opt.ShardPicker != nil {
if len(state.Masters) == 0 {
return errClusterNoNodes
}
// For read-only keyless commands, pick from all nodes (masters + slaves)
allNodes := append(state.Masters, state.Slaves...)
idx := c.opt.ShardPicker.Next(len(allNodes))
node = allNodes[idx]
} else {
node, err = c.slotReadOnlyNode(state, slot)
}
if err != nil {
return err
if err != nil {
return err
}
}
cmdsMap.Add(node, cmd)
}
@@ -1518,14 +1608,18 @@ func (c *ClusterClient) mapCmdsByNode(ctx context.Context, cmdsMap *cmdsMap, cmd
}
slot := c.cmdSlot(cmd, -1)
var node *clusterNode
if slot == -1 {
// Keyless command - use ShardPicker to select a node
node, err = c.cmdNodeWithShardPicker(ctx, cmd.Name(), slot, c.opt.ShardPicker)
// For keyless commands (slot == -1), use ShardPicker if routing policies are enabled
if slot == -1 && !c.opt.DisableRoutingPolicies && c.opt.ShardPicker != nil {
if len(state.Masters) == 0 {
return errClusterNoNodes
}
idx := c.opt.ShardPicker.Next(len(state.Masters))
node = state.Masters[idx]
} else {
node, err = state.slotMasterNode(slot)
}
if err != nil {
return err
if err != nil {
return err
}
}
cmdsMap.Add(node, cmd)
}
osscluster_lazy_reload_test.go
+201
View File
@@ -0,0 +1,201 @@
package redis
import (
"context"
"sync/atomic"
"testing"
"time"
)
// TestLazyReloadQueueBehavior tests that LazyReload properly queues reload requests
func TestLazyReloadQueueBehavior(t *testing.T) {
t.Run("SingleReload", func(t *testing.T) {
var reloadCount atomic.Int32
holder := newClusterStateHolder(func(ctx context.Context) (*clusterState, error) {
reloadCount.Add(1)
time.Sleep(50 * time.Millisecond) // Simulate reload work
return &clusterState{}, nil
}, 10*time.Second)
// Trigger one reload
holder.LazyReload()
// Wait for reload to complete
time.Sleep(300 * time.Millisecond)
if count := reloadCount.Load(); count != 1 {
t.Errorf("Expected 1 reload, got %d", count)
}
})
t.Run("ConcurrentReloadsDeduplication", func(t *testing.T) {
var reloadCount atomic.Int32
holder := newClusterStateHolder(func(ctx context.Context) (*clusterState, error) {
reloadCount.Add(1)
time.Sleep(50 * time.Millisecond) // Simulate reload work
return &clusterState{}, nil
}, 10*time.Second)
// Trigger multiple reloads concurrently
for i := 0; i < 10; i++ {
go holder.LazyReload()
}
// Wait for all to complete
time.Sleep(100 * time.Millisecond)
// Should only reload once (all concurrent calls deduplicated)
if count := reloadCount.Load(); count != 1 {
t.Errorf("Expected 1 reload (deduplication), got %d", count)
}
})
t.Run("PendingReloadDuringCooldown", func(t *testing.T) {
var reloadCount atomic.Int32
holder := newClusterStateHolder(func(ctx context.Context) (*clusterState, error) {
reloadCount.Add(1)
time.Sleep(10 * time.Millisecond) // Simulate reload work
return &clusterState{}, nil
}, 10*time.Second)
// Trigger first reload
holder.LazyReload()
// Wait for reload to complete but still in cooldown
time.Sleep(50 * time.Millisecond)
// Trigger second reload during cooldown period
holder.LazyReload()
// Wait for second reload to complete
time.Sleep(300 * time.Millisecond)
// Should have reloaded twice (second request queued and executed)
if count := reloadCount.Load(); count != 2 {
t.Errorf("Expected 2 reloads (queued during cooldown), got %d", count)
}
})
t.Run("MultiplePendingReloadsCollapsed", func(t *testing.T) {
var reloadCount atomic.Int32
holder := newClusterStateHolder(func(ctx context.Context) (*clusterState, error) {
reloadCount.Add(1)
time.Sleep(10 * time.Millisecond) // Simulate reload work
return &clusterState{}, nil
}, 10*time.Second)
// Trigger first reload
holder.LazyReload()
// Wait for reload to start
time.Sleep(5 * time.Millisecond)
// Trigger multiple reloads during active reload + cooldown
for i := 0; i < 10; i++ {
holder.LazyReload()
time.Sleep(5 * time.Millisecond)
}
// Wait for all to complete
time.Sleep(400 * time.Millisecond)
// Should have reloaded exactly twice:
// 1. Initial reload
// 2. One more reload for all the pending requests (collapsed into one)
if count := reloadCount.Load(); count != 2 {
t.Errorf("Expected 2 reloads (initial + collapsed pending), got %d", count)
}
})
t.Run("ReloadAfterCooldownPeriod", func(t *testing.T) {
var reloadCount atomic.Int32
holder := newClusterStateHolder(func(ctx context.Context) (*clusterState, error) {
reloadCount.Add(1)
time.Sleep(10 * time.Millisecond) // Simulate reload work
return &clusterState{}, nil
}, 10*time.Second)
// Trigger first reload
holder.LazyReload()
// Wait for reload + cooldown to complete
time.Sleep(300 * time.Millisecond)
// Trigger second reload after cooldown
holder.LazyReload()
// Wait for second reload to complete
time.Sleep(300 * time.Millisecond)
// Should have reloaded twice (separate reload cycles)
if count := reloadCount.Load(); count != 2 {
t.Errorf("Expected 2 reloads (separate cycles), got %d", count)
}
})
t.Run("ErrorDuringReload", func(t *testing.T) {
var reloadCount atomic.Int32
var shouldFail atomic.Bool
shouldFail.Store(true)
holder := newClusterStateHolder(func(ctx context.Context) (*clusterState, error) {
reloadCount.Add(1)
if shouldFail.Load() {
return nil, context.DeadlineExceeded
}
return &clusterState{}, nil
}, 10*time.Second)
// Trigger reload that will fail
holder.LazyReload()
// Wait for failed reload
time.Sleep(50 * time.Millisecond)
// Trigger another reload (should succeed now)
shouldFail.Store(false)
holder.LazyReload()
// Wait for successful reload
time.Sleep(300 * time.Millisecond)
// Should have attempted reload twice (first failed, second succeeded)
if count := reloadCount.Load(); count != 2 {
t.Errorf("Expected 2 reload attempts, got %d", count)
}
})
t.Run("CascadingSMIGRATEDScenario", func(t *testing.T) {
// Simulate the real-world scenario: multiple SMIGRATED notifications
// arriving in quick succession from different node clients
var reloadCount atomic.Int32
holder := newClusterStateHolder(func(ctx context.Context) (*clusterState, error) {
reloadCount.Add(1)
time.Sleep(20 * time.Millisecond) // Simulate realistic reload time
return &clusterState{}, nil
}, 10*time.Second)
// Simulate 5 SMIGRATED notifications arriving within 100ms
for i := 0; i < 5; i++ {
go holder.LazyReload()
time.Sleep(20 * time.Millisecond)
}
// Wait for all reloads to complete
time.Sleep(500 * time.Millisecond)
// Should reload at most 2 times:
// 1. First notification triggers reload
// 2. Notifications 2-5 collapse into one pending reload
count := reloadCount.Load()
if count < 1 || count > 2 {
t.Errorf("Expected 1-2 reloads for cascading scenario, got %d", count)
}
})
}
osscluster_maintnotifications_test.go
+692
View File
@@ -0,0 +1,692 @@
package redis_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/redis/go-redis/v9"
)
// multiNodeProxy represents a cae-resp-proxy instance that can mimic multiple cluster nodes
type multiNodeProxy struct {
apiPort int
apiBaseURL string
cmd *exec.Cmd
httpClient *http.Client
nodes []proxyNode
}
// proxyNode represents a single node in the multi-node proxy
type proxyNode struct {
listenPort int
targetHost string
targetPort int
proxyAddr string
nodeID string
}
// newMultiNodeProxy creates a new multi-node proxy instance
func newMultiNodeProxy(apiPort int) *multiNodeProxy {
return &multiNodeProxy{
apiPort: apiPort,
apiBaseURL: fmt.Sprintf("http://localhost:%d", apiPort),
httpClient: &http.Client{
Timeout: 5 * time.Second,
},
nodes: make([]proxyNode, 0),
}
}
// start starts the proxy server with initial node
func (mp *multiNodeProxy) start(initialListenPort, targetPort int, targetHost string) error {
// Start cae-resp-proxy with just the API port
// We'll add nodes dynamically via the API
mp.cmd = exec.Command("cae-resp-proxy",
"--api-port", fmt.Sprintf("%d", mp.apiPort),
"--listen-port", fmt.Sprintf("%d", initialListenPort),
"--target-host", targetHost,
"--target-port", fmt.Sprintf("%d", targetPort),
)
if err := mp.cmd.Start(); err != nil {
return fmt.Errorf("failed to start proxy: %w", err)
}
// Wait for proxy to be ready
time.Sleep(500 * time.Millisecond)
// Verify proxy is responding
for i := 0; i < 10; i++ {
resp, err := mp.httpClient.Get(mp.apiBaseURL + "/stats")
if err == nil {
resp.Body.Close()
if resp.StatusCode == 200 {
// Add the initial node to our tracking
mp.nodes = append(mp.nodes, proxyNode{
listenPort: initialListenPort,
targetHost: targetHost,
targetPort: targetPort,
proxyAddr: fmt.Sprintf("localhost:%d", initialListenPort),
nodeID: fmt.Sprintf("localhost:%d:%d", initialListenPort, targetPort),
})
return nil
}
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("proxy did not become ready")
}
// addNode adds a new proxy node dynamically
func (mp *multiNodeProxy) addNode(listenPort, targetPort int, targetHost string) (*proxyNode, error) {
// Use the /nodes API to add a new proxy node
nodeConfig := map[string]interface{}{
"listenPort": listenPort,
"targetHost": targetHost,
"targetPort": targetPort,
}
jsonData, err := json.Marshal(nodeConfig)
if err != nil {
return nil, fmt.Errorf("failed to marshal node config: %w", err)
}
resp, err := mp.httpClient.Post(
mp.apiBaseURL+"/nodes",
"application/json",
bytes.NewReader(jsonData),
)
if err != nil {
return nil, fmt.Errorf("failed to add node: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("failed to add node, status %d: %s", resp.StatusCode, string(body))
}
// Create node tracking
node := proxyNode{
listenPort: listenPort,
targetHost: targetHost,
targetPort: targetPort,
proxyAddr: fmt.Sprintf("localhost:%d", listenPort),
nodeID: fmt.Sprintf("localhost:%d:%d", listenPort, targetPort),
}
mp.nodes = append(mp.nodes, node)
// Wait a bit for the node to be ready
time.Sleep(200 * time.Millisecond)
return &node, nil
}
// getNodes returns all proxy nodes
func (mp *multiNodeProxy) getNodes() []proxyNode {
return mp.nodes
}
// getNodeAddrs returns all proxy node addresses for cluster client
func (mp *multiNodeProxy) getNodeAddrs() []string {
addrs := make([]string, len(mp.nodes))
for i, node := range mp.nodes {
addrs[i] = node.proxyAddr
}
return addrs
}
// stop stops the proxy server
func (mp *multiNodeProxy) stop() error {
if mp.cmd != nil && mp.cmd.Process != nil {
return mp.cmd.Process.Kill()
}
return nil
}
// injectNotification injects a RESP3 push notification to all connected clients
func (mp *multiNodeProxy) injectNotification(notification string) error {
url := mp.apiBaseURL + "/send-to-all-clients?encoding=raw"
resp, err := mp.httpClient.Post(url, "application/octet-stream", strings.NewReader(notification))
if err != nil {
return fmt.Errorf("failed to inject notification: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("injection failed with status %d: %s", resp.StatusCode, string(body))
}
return nil
}
// injectNotificationToNode injects a notification to clients connected to a specific node
func (mp *multiNodeProxy) injectNotificationToNode(nodeAddr string, notification string) error {
// Get all connections
resp, err := mp.httpClient.Get(mp.apiBaseURL + "/connections")
if err != nil {
return fmt.Errorf("failed to get connections: %w", err)
}
defer resp.Body.Close()
var connResp struct {
ConnectionIDs []string `json:"connectionIds"`
}
if err := json.NewDecoder(resp.Body).Decode(&connResp); err != nil {
return fmt.Errorf("failed to decode connections: %w", err)
}
// For simplicity, inject to all clients
// In a real scenario, you'd filter by node
return mp.injectNotification(notification)
}
// formatSMigratingNotification creates a SMIGRATING push notification in RESP3 format
// Format: ["SMIGRATING", SeqID, slot/range1-range2, ...]
func formatSMigratingNotification(seqID int64, slots ...string) string {
// >N\r\n where N is the number of elements
parts := []string{fmt.Sprintf(">%d\r\n", 2+len(slots))}
// $10\r\nSMIGRATING\r\n
parts = append(parts, "$10\r\nSMIGRATING\r\n")
// :seqID\r\n
parts = append(parts, fmt.Sprintf(":%d\r\n", seqID))
// Add each slot/range as bulk string
for _, slot := range slots {
parts = append(parts, fmt.Sprintf("$%d\r\n%s\r\n", len(slot), slot))
}
return strings.Join(parts, "")
}
// formatSMigratedNotification creates a SMIGRATED push notification in RESP3 format
// RESP3 wire format:
//
// >3 <- push frame with 3 top-level elements
// +SMIGRATED <- message name
// :SeqID <- sequence id integer
// *<num_entries> <- array of triplet arrays
// *3 <- each triplet is a 3-element array
// +<source> <- node from which slots are migrating FROM
// +<destination> <- node to which slots are migrating TO
// +<slots> <- comma-separated slots and/or ranges
//
// Each triplet is formatted as: "source target slots"
// Example: "abc.com:6789 abc.com:6790 123,789-1000"
func formatSMigratedNotification(seqID int64, triplets ...string) string {
parts := []string{">3\r\n"}
// +SMIGRATED\r\n
parts = append(parts, "+SMIGRATED\r\n")
// :seqID\r\n
parts = append(parts, fmt.Sprintf(":%d\r\n", seqID))
// Outer array containing all triplets
parts = append(parts, fmt.Sprintf("*%d\r\n", len(triplets)))
for _, triplet := range triplets {
// Split triplet into source, target, and slots
tripletParts := strings.SplitN(triplet, " ", 3)
if len(tripletParts) != 3 {
continue
}
source := tripletParts[0]
target := tripletParts[1]
slots := tripletParts[2]
// Each triplet is a 3-element array
parts = append(parts, "*3\r\n")
parts = append(parts, fmt.Sprintf("+%s\r\n", source))
parts = append(parts, fmt.Sprintf("+%s\r\n", target))
parts = append(parts, fmt.Sprintf("+%s\r\n", slots))
}
return strings.Join(parts, "")
}
// TestClusterMaintNotifications_SMIGRATING tests SMIGRATING notification handling
func TestClusterMaintNotifications_SMIGRATING(t *testing.T) {
if os.Getenv("CLUSTER_MAINT_INTEGRATION_TEST") != "true" {
t.Skip("Skipping cluster maintnotifications integration test. Set CLUSTER_MAINT_INTEGRATION_TEST=true to run")
}
ctx := context.Background()
// Create multi-node proxy that mimics a 3-node cluster
proxy := newMultiNodeProxy(8000)
if err := proxy.start(7000, 6379, "localhost"); err != nil {
t.Fatalf("Failed to start proxy: %v", err)
}
defer proxy.stop()
// Add two more nodes to mimic a 3-node cluster
if _, err := proxy.addNode(7001, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node 2: %v", err)
}
if _, err := proxy.addNode(7002, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node 3: %v", err)
}
t.Logf("Started proxy with %d nodes: %v", len(proxy.getNodes()), proxy.getNodeAddrs())
// Create cluster client pointing to all proxy nodes
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: proxy.getNodeAddrs(),
Protocol: 3, // RESP3 required for push notifications
})
defer client.Close()
// Verify connection works
if err := client.Ping(ctx).Err(); err != nil {
t.Fatalf("Failed to connect to cluster via proxy: %v", err)
}
// Perform some operations to establish connections
for i := 0; i < 5; i++ {
if err := client.Set(ctx, fmt.Sprintf("key%d", i), "value", 0).Err(); err != nil {
t.Logf("Warning: Failed to set key: %v", err)
}
}
// Inject SMIGRATING notification to all nodes
notification := formatSMigratingNotification(12345, "1000", "2000-3000")
if err := proxy.injectNotification(notification); err != nil {
t.Fatalf("Failed to inject SMIGRATING notification: %v", err)
}
// Wait for notification processing
time.Sleep(200 * time.Millisecond)
// Verify operations still work (timeouts should be relaxed)
if err := client.Set(ctx, "test-key-during-migration", "value", 0).Err(); err != nil {
t.Errorf("Expected operations to work during migration, got error: %v", err)
}
t.Log("SMIGRATING notification test passed")
}
// TestClusterMaintNotifications_SMIGRATED tests SMIGRATED notification handling and cluster state reload
func TestClusterMaintNotifications_SMIGRATED(t *testing.T) {
if os.Getenv("CLUSTER_MAINT_INTEGRATION_TEST") != "true" {
t.Skip("Skipping cluster maintnotifications integration test. Set CLUSTER_MAINT_INTEGRATION_TEST=true to run")
}
ctx := context.Background()
// Create multi-node proxy
proxy := newMultiNodeProxy(8001)
if err := proxy.start(7010, 6379, "localhost"); err != nil {
t.Fatalf("Failed to start proxy: %v", err)
}
defer proxy.stop()
// Add more nodes
if _, err := proxy.addNode(7011, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node 2: %v", err)
}
if _, err := proxy.addNode(7012, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node 3: %v", err)
}
t.Logf("Started proxy with %d nodes: %v", len(proxy.getNodes()), proxy.getNodeAddrs())
// Track cluster state reloads
var reloadCount atomic.Int32
// Create cluster client
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: proxy.getNodeAddrs(),
Protocol: 3,
})
defer client.Close()
// Hook to track state reloads via callback
client.OnNewNode(func(nodeClient *redis.Client) {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.SetClusterStateReloadCallback(func(ctx context.Context, hostPort string, slotRanges []string) {
reloadCount.Add(1)
})
}
})
// Verify connection
if err := client.Ping(ctx).Err(); err != nil {
t.Fatalf("Failed to connect: %v", err)
}
// Perform operations to establish connections
for i := 0; i < 5; i++ {
client.Set(ctx, fmt.Sprintf("key%d", i), "value", 0)
}
initialReloads := reloadCount.Load()
// Inject SMIGRATED notification with new format
// Simulate migration from node 1 to node 2
notification := formatSMigratedNotification(12346, "127.0.0.1:7011 1000,2000-3000")
if err := proxy.injectNotification(notification); err != nil {
t.Fatalf("Failed to inject SMIGRATED notification: %v", err)
}
// Wait for notification processing and state reload
time.Sleep(500 * time.Millisecond)
// Verify cluster state was reloaded
finalReloads := reloadCount.Load()
if finalReloads <= initialReloads {
t.Errorf("Expected cluster state reload after SMIGRATED, reloads: initial=%d, final=%d",
initialReloads, finalReloads)
}
// Verify operations still work
if err := client.Set(ctx, "test-key-after-smigrated", "value", 0).Err(); err != nil {
t.Errorf("Expected operations to work after SMIGRATED, got error: %v", err)
}
t.Log("SMIGRATED notification test passed")
}
// TestClusterMaintNotifications_Deduplication tests that SMIGRATED notifications are deduplicated by SeqID
func TestClusterMaintNotifications_Deduplication(t *testing.T) {
if os.Getenv("CLUSTER_MAINT_INTEGRATION_TEST") != "true" {
t.Skip("Skipping cluster maintnotifications integration test. Set CLUSTER_MAINT_INTEGRATION_TEST=true to run")
}
ctx := context.Background()
// Create multi-node proxy
proxy := newMultiNodeProxy(8002)
if err := proxy.start(7020, 6379, "localhost"); err != nil {
t.Fatalf("Failed to start proxy: %v", err)
}
defer proxy.stop()
// Add more nodes
if _, err := proxy.addNode(7021, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node 2: %v", err)
}
if _, err := proxy.addNode(7022, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node 3: %v", err)
}
t.Logf("Started proxy with %d nodes: %v", len(proxy.getNodes()), proxy.getNodeAddrs())
var reloadCount atomic.Int32
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: proxy.getNodeAddrs(),
Protocol: 3,
})
defer client.Close()
// Track reloads via callback
client.OnNewNode(func(nodeClient *redis.Client) {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.SetClusterStateReloadCallback(func(ctx context.Context, hostPort string, slotRanges []string) {
reloadCount.Add(1)
})
}
})
// Verify connection
if err := client.Ping(ctx).Err(); err != nil {
t.Fatalf("Failed to connect: %v", err)
}
// Perform operations
for i := 0; i < 5; i++ {
client.Set(ctx, fmt.Sprintf("key%d", i), "value", 0)
}
initialReloads := reloadCount.Load()
// Inject the same SMIGRATED notification multiple times (same SeqID)
// This simulates receiving the same notification from multiple nodes
seqID := int64(99999)
notification := formatSMigratedNotification(seqID, "127.0.0.1:7021 5000-6000")
// Inject 5 times to all nodes
for i := 0; i < 5; i++ {
if err := proxy.injectNotification(notification); err != nil {
t.Fatalf("Failed to inject notification: %v", err)
}
time.Sleep(50 * time.Millisecond)
}
// Wait for processing
time.Sleep(500 * time.Millisecond)
// Verify only ONE reload happened (deduplication by SeqID)
finalReloads := reloadCount.Load()
reloadDiff := finalReloads - initialReloads
if reloadDiff != 1 {
t.Errorf("Expected exactly 1 reload due to deduplication, got %d reloads", reloadDiff)
}
t.Log("Deduplication test passed")
}
// TestClusterMaintNotifications_MultiNode tests notifications across multiple cluster nodes
func TestClusterMaintNotifications_MultiNode(t *testing.T) {
if os.Getenv("CLUSTER_MAINT_INTEGRATION_TEST") != "true" {
t.Skip("Skipping cluster maintnotifications integration test. Set CLUSTER_MAINT_INTEGRATION_TEST=true to run")
}
ctx := context.Background()
// Create multi-node proxy with 5 nodes to mimic a real cluster
proxy := newMultiNodeProxy(8003)
if err := proxy.start(7030, 6379, "localhost"); err != nil {
t.Fatalf("Failed to start proxy: %v", err)
}
defer proxy.stop()
// Add 4 more nodes for a 5-node cluster
for i := 1; i < 5; i++ {
if _, err := proxy.addNode(7030+i, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node %d: %v", i+1, err)
}
}
nodes := proxy.getNodes()
t.Logf("Started proxy with %d nodes: %v", len(nodes), proxy.getNodeAddrs())
// Track notifications received
var migratingCount atomic.Int32
var migratedCount atomic.Int32
var reloadCount atomic.Int32
// Create cluster client
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: proxy.getNodeAddrs(),
Protocol: 3,
})
defer client.Close()
// Set up tracking
client.OnNewNode(func(nodeClient *redis.Client) {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.SetClusterStateReloadCallback(func(ctx context.Context, hostPort string, slotRanges []string) {
reloadCount.Add(1)
t.Logf("Cluster state reload triggered for %s, slots: %v", hostPort, slotRanges)
})
}
})
// Verify connection
if err := client.Ping(ctx).Err(); err != nil {
t.Fatalf("Failed to connect: %v", err)
}
// Perform operations to establish connections to all nodes
for i := 0; i < 20; i++ {
client.Set(ctx, fmt.Sprintf("key%d", i), "value", 0)
}
time.Sleep(500 * time.Millisecond)
// Simulate slot migration scenario:
// 1. SMIGRATING: Slots 0-5000 are migrating from node 1 to node 2
t.Log("Injecting SMIGRATING notification...")
migratingNotif := formatSMigratingNotification(10001, "0-5000")
if err := proxy.injectNotification(migratingNotif); err != nil {
t.Fatalf("Failed to inject SMIGRATING: %v", err)
}
migratingCount.Add(1)
time.Sleep(300 * time.Millisecond)
// 2. SMIGRATED: Migration completed, slots now on node 2
t.Log("Injecting SMIGRATED notification...")
migratedNotif := formatSMigratedNotification(10002,
fmt.Sprintf("127.0.0.1:%d 0-5000", nodes[1].listenPort))
if err := proxy.injectNotification(migratedNotif); err != nil {
t.Fatalf("Failed to inject SMIGRATED: %v", err)
}
migratedCount.Add(1)
time.Sleep(500 * time.Millisecond)
// 3. Another migration: Slots 5001-10000 from node 2 to node 3
t.Log("Injecting second SMIGRATING notification...")
migratingNotif2 := formatSMigratingNotification(10003, "5001-10000")
if err := proxy.injectNotification(migratingNotif2); err != nil {
t.Fatalf("Failed to inject second SMIGRATING: %v", err)
}
migratingCount.Add(1)
time.Sleep(300 * time.Millisecond)
// 4. Second migration completed
t.Log("Injecting second SMIGRATED notification...")
migratedNotif2 := formatSMigratedNotification(10004,
fmt.Sprintf("127.0.0.1:%d 5001-10000", nodes[2].listenPort))
if err := proxy.injectNotification(migratedNotif2); err != nil {
t.Fatalf("Failed to inject second SMIGRATED: %v", err)
}
migratedCount.Add(1)
time.Sleep(500 * time.Millisecond)
// Verify cluster state was reloaded for each SMIGRATED
finalReloads := reloadCount.Load()
if finalReloads < 2 {
t.Errorf("Expected at least 2 cluster state reloads, got %d", finalReloads)
}
// Verify operations still work after migrations
for i := 0; i < 10; i++ {
if err := client.Set(ctx, fmt.Sprintf("post-migration-key%d", i), "value", 0).Err(); err != nil {
t.Errorf("Expected operations to work after migrations, got error: %v", err)
}
}
t.Logf("Multi-node test passed: SMIGRATING=%d, SMIGRATED=%d, Reloads=%d",
migratingCount.Load(), migratedCount.Load(), finalReloads)
}
// TestClusterMaintNotifications_ComplexMigration tests complex multi-endpoint migration
func TestClusterMaintNotifications_ComplexMigration(t *testing.T) {
if os.Getenv("CLUSTER_MAINT_INTEGRATION_TEST") != "true" {
t.Skip("Skipping cluster maintnotifications integration test. Set CLUSTER_MAINT_INTEGRATION_TEST=true to run")
}
ctx := context.Background()
// Create multi-node proxy
proxy := newMultiNodeProxy(8004)
if err := proxy.start(7040, 6379, "localhost"); err != nil {
t.Fatalf("Failed to start proxy: %v", err)
}
defer proxy.stop()
// Add 2 more nodes
if _, err := proxy.addNode(7041, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node 2: %v", err)
}
if _, err := proxy.addNode(7042, 6379, "localhost"); err != nil {
t.Fatalf("Failed to add node 3: %v", err)
}
nodes := proxy.getNodes()
t.Logf("Started proxy with %d nodes: %v", len(nodes), proxy.getNodeAddrs())
var reloadCount atomic.Int32
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: proxy.getNodeAddrs(),
Protocol: 3,
})
defer client.Close()
client.OnNewNode(func(nodeClient *redis.Client) {
manager := nodeClient.GetMaintNotificationsManager()
if manager != nil {
manager.SetClusterStateReloadCallback(func(ctx context.Context, hostPort string, slotRanges []string) {
reloadCount.Add(1)
t.Logf("Reload for %s, slots: %v", hostPort, slotRanges)
})
}
})
if err := client.Ping(ctx).Err(); err != nil {
t.Fatalf("Failed to connect: %v", err)
}
// Perform operations
for i := 0; i < 10; i++ {
client.Set(ctx, fmt.Sprintf("key%d", i), "value", 0)
}
initialReloads := reloadCount.Load()
// Test new SMIGRATED format with multiple endpoints
// Simulate a complex resharding where slots are distributed to multiple nodes
t.Log("Injecting complex SMIGRATED notification with multiple endpoints...")
notification := formatSMigratedNotification(20001,
fmt.Sprintf("127.0.0.1:%d 0-5000,10000-12000", nodes[0].listenPort),
fmt.Sprintf("127.0.0.1:%d 5001-9999", nodes[1].listenPort),
fmt.Sprintf("127.0.0.1:%d 12001-16383", nodes[2].listenPort),
)
if err := proxy.injectNotification(notification); err != nil {
t.Fatalf("Failed to inject complex SMIGRATED: %v", err)
}
time.Sleep(500 * time.Millisecond)
// Verify reload happened
finalReloads := reloadCount.Load()
if finalReloads <= initialReloads {
t.Errorf("Expected cluster state reload, reloads: initial=%d, final=%d",
initialReloads, finalReloads)
}
// Verify operations work
for i := 0; i < 10; i++ {
if err := client.Set(ctx, fmt.Sprintf("complex-key%d", i), "value", 0).Err(); err != nil {
t.Errorf("Operations failed after complex migration: %v", err)
}
}
t.Log("Complex migration test passed")
}
osscluster_maintnotifications_unit_test.go
+345
View File
@@ -0,0 +1,345 @@
package redis_test
import (
"context"
"sync/atomic"
"testing"
"time"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/maintnotifications"
)
// TestClusterMaintNotifications_CallbackSetup tests that the cluster state reload callback is properly set up
func TestClusterMaintNotifications_CallbackSetup(t *testing.T) {
// Create a mock cluster with maintnotifications enabled
opt := &redis.ClusterOptions{
Addrs: []string{"localhost:6379"},
Protocol: 3,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeEnabled,
},
// Use a custom ClusterSlots function to avoid needing a real cluster
ClusterSlots: func(ctx context.Context) ([]redis.ClusterSlot, error) {
return []redis.ClusterSlot{
{
Start: 0,
End: 16383,
Nodes: []redis.ClusterNode{
{Addr: "localhost:6379"},
},
},
}, nil
},
}
client := redis.NewClusterClient(opt)
defer client.Close()
// Give time for initialization
time.Sleep(100 * time.Millisecond)
// Verify that maintnotifications manager is available on node clients
// We can't directly access the manager, but we can verify the setup worked
// by checking that the client was created successfully
if client == nil {
t.Fatal("Expected cluster client to be created")
}
t.Log("Cluster maintnotifications callback setup test passed")
}
// TestClusterMaintNotifications_SMigratingHandler tests SMIGRATING notification handling
func TestClusterMaintNotifications_SMigratingHandler(t *testing.T) {
// Simulate receiving a SMIGRATING notification
notification := []interface{}{
"SMIGRATING",
int64(12345),
"1000",
"2000-3000",
}
// In a real scenario, this would be handled by the NotificationHandler
// For unit testing, we verify the notification format is correct
if len(notification) < 3 {
t.Fatal("SMIGRATING notification should have at least 3 elements")
}
notifType, ok := notification[0].(string)
if !ok || notifType != "SMIGRATING" {
t.Fatalf("Expected notification type SMIGRATING, got %v", notification[0])
}
seqID, ok := notification[1].(int64)
if !ok {
t.Fatalf("Expected SeqID to be int64, got %T", notification[1])
}
if seqID != 12345 {
t.Errorf("Expected SeqID 12345, got %d", seqID)
}
// Verify slot ranges
if len(notification) < 3 {
t.Fatal("Expected at least one slot range")
}
slot1, ok := notification[2].(string)
if !ok || slot1 != "1000" {
t.Errorf("Expected first slot to be '1000', got %v", notification[2])
}
if len(notification) >= 4 {
slot2, ok := notification[3].(string)
if !ok || slot2 != "2000-3000" {
t.Errorf("Expected second slot range to be '2000-3000', got %v", notification[3])
}
}
t.Log("SMIGRATING notification format validation passed")
}
// TestClusterMaintNotifications_SMigratedHandler tests SMIGRATED notification handling
func TestClusterMaintNotifications_SMigratedHandler(t *testing.T) {
// Simulate receiving a SMIGRATED notification with nested array format
// Format: ["SMIGRATED", SeqID, [[source, target, slots], [source, target, slots], ...]]
notification := []interface{}{
"SMIGRATED",
int64(12346),
[]interface{}{
[]interface{}{"127.0.0.1:6379", "127.0.0.1:6380", "123,456,789-1000"},
[]interface{}{"127.0.0.1:6379", "127.0.0.1:6381", "124,457,300-500"},
},
}
// Verify notification format: SMIGRATED + SeqID + triplets array = 3 elements
if len(notification) != 3 {
t.Fatalf("SMIGRATED notification should have exactly 3 elements, got %d", len(notification))
}
notifType, ok := notification[0].(string)
if !ok || notifType != "SMIGRATED" {
t.Fatalf("Expected notification type SMIGRATED, got %v", notification[0])
}
seqID, ok := notification[1].(int64)
if !ok {
t.Fatalf("Expected SeqID to be int64, got %T", notification[1])
}
if seqID != 12346 {
t.Errorf("Expected SeqID 12346, got %d", seqID)
}
// Verify triplets array
triplets, ok := notification[2].([]interface{})
if !ok {
t.Fatalf("Expected triplets to be array, got %T", notification[2])
}
if len(triplets) != 2 {
t.Fatalf("Expected 2 triplets, got %d", len(triplets))
}
// Verify first triplet (source, target, slots)
triplet1, ok := triplets[0].([]interface{})
if !ok || len(triplet1) != 3 {
t.Fatalf("Expected first triplet to be 3-element array, got %v", triplets[0])
}
source1, ok := triplet1[0].(string)
if !ok || source1 != "127.0.0.1:6379" {
t.Errorf("Expected first triplet source '127.0.0.1:6379', got %v", triplet1[0])
}
target1, ok := triplet1[1].(string)
if !ok || target1 != "127.0.0.1:6380" {
t.Errorf("Expected first triplet target '127.0.0.1:6380', got %v", triplet1[1])
}
slots1, ok := triplet1[2].(string)
if !ok || slots1 != "123,456,789-1000" {
t.Errorf("Expected first triplet slots '123,456,789-1000', got %v", triplet1[2])
}
// Verify second triplet
triplet2, ok := triplets[1].([]interface{})
if !ok || len(triplet2) != 3 {
t.Fatalf("Expected second triplet to be 3-element array, got %v", triplets[1])
}
source2, ok := triplet2[0].(string)
if !ok || source2 != "127.0.0.1:6379" {
t.Errorf("Expected second triplet source '127.0.0.1:6379', got %v", triplet2[0])
}
target2, ok := triplet2[1].(string)
if !ok || target2 != "127.0.0.1:6381" {
t.Errorf("Expected second triplet target '127.0.0.1:6381', got %v", triplet2[1])
}
slots2, ok := triplet2[2].(string)
if !ok || slots2 != "124,457,300-500" {
t.Errorf("Expected second triplet slots '124,457,300-500', got %v", triplet2[2])
}
t.Log("SMIGRATED notification format validation passed")
}
// TestClusterMaintNotifications_DeduplicationLogic tests the deduplication logic for SMIGRATED
func TestClusterMaintNotifications_DeduplicationLogic(t *testing.T) {
// This test verifies the deduplication concept
// In the actual implementation, SMIGRATED notifications with the same SeqID
// should only trigger cluster state reload once
processedSeqIDs := make(map[int64]bool)
var reloadCount int
// Simulate receiving multiple SMIGRATED notifications with same SeqID
notifications := []int64{12345, 12345, 12345, 12346, 12346, 12347}
for _, seqID := range notifications {
// Check if already processed
if !processedSeqIDs[seqID] {
processedSeqIDs[seqID] = true
reloadCount++
}
}
// Should have 3 unique SeqIDs (12345, 12346, 12347)
if reloadCount != 3 {
t.Errorf("Expected 3 unique reloads, got %d", reloadCount)
}
if len(processedSeqIDs) != 3 {
t.Errorf("Expected 3 unique SeqIDs, got %d", len(processedSeqIDs))
}
t.Log("Deduplication logic test passed")
}
// TestClusterMaintNotifications_NotificationTypes tests that cluster notification types are defined
func TestClusterMaintNotifications_NotificationTypes(t *testing.T) {
// Verify that SMIGRATING and SMIGRATED constants exist
// These are defined in maintnotifications package
expectedTypes := []string{
maintnotifications.NotificationSMigrating,
maintnotifications.NotificationSMigrated,
}
for _, notifType := range expectedTypes {
if notifType == "" {
t.Errorf("Notification type should not be empty")
}
}
// Verify the values
if maintnotifications.NotificationSMigrating != "SMIGRATING" {
t.Errorf("Expected SMIGRATING, got %s", maintnotifications.NotificationSMigrating)
}
if maintnotifications.NotificationSMigrated != "SMIGRATED" {
t.Errorf("Expected SMIGRATED, got %s", maintnotifications.NotificationSMigrated)
}
t.Log("Notification types test passed")
}
// TestClusterMaintNotifications_ConfigValidation tests maintnotifications config for cluster
func TestClusterMaintNotifications_ConfigValidation(t *testing.T) {
// Test valid config
config := &maintnotifications.Config{
Mode: maintnotifications.ModeEnabled,
RelaxedTimeout: 10 * time.Second,
}
if err := config.ApplyDefaults().Validate(); err != nil {
t.Errorf("Valid config should pass validation: %v", err)
}
// Test that config can be cloned (important for cluster where each node gets a copy)
cloned := config.Clone()
if cloned.Mode != config.Mode {
t.Error("Cloned config should have same mode")
}
if cloned.RelaxedTimeout != config.RelaxedTimeout {
t.Error("Cloned config should have same relaxed timeout")
}
// Modify original to ensure clone is independent
config.RelaxedTimeout = 20 * time.Second
if cloned.RelaxedTimeout == config.RelaxedTimeout {
t.Error("Clone should be independent of original")
}
t.Log("Config validation test passed")
}
// TestClusterMaintNotifications_StateReloadCallback tests the callback mechanism
func TestClusterMaintNotifications_StateReloadCallback(t *testing.T) {
var callbackInvoked atomic.Bool
var receivedHostPort atomic.Value
var receivedSlots atomic.Value
// Simulate the callback that would be set on the manager
callback := func(ctx context.Context, hostPort string, slotRanges []string) {
callbackInvoked.Store(true)
receivedHostPort.Store(hostPort)
receivedSlots.Store(slotRanges)
}
// Simulate invoking the callback (as would happen when SMIGRATED is received)
ctx := context.Background()
callback(ctx, "127.0.0.1:6380", []string{"1000", "2000-3000"})
// Verify callback was invoked
if !callbackInvoked.Load() {
t.Error("Expected callback to be invoked")
}
// Verify parameters
hostPort := receivedHostPort.Load().(string)
if hostPort != "127.0.0.1:6380" {
t.Errorf("Expected host:port '127.0.0.1:6380', got %s", hostPort)
}
slots := receivedSlots.Load().([]string)
if len(slots) != 2 {
t.Errorf("Expected 2 slot ranges, got %d", len(slots))
}
if slots[0] != "1000" {
t.Errorf("Expected first slot '1000', got %s", slots[0])
}
if slots[1] != "2000-3000" {
t.Errorf("Expected second slot range '2000-3000', got %s", slots[1])
}
t.Log("State reload callback test passed")
}
// TestClusterMaintNotifications_ConcurrentCallbacks tests concurrent callback invocations
func TestClusterMaintNotifications_ConcurrentCallbacks(t *testing.T) {
var callbackCount atomic.Int32
callback := func(ctx context.Context, hostPort string, slotRanges []string) {
callbackCount.Add(1)
// Simulate some processing time
time.Sleep(10 * time.Millisecond)
}
// Invoke callback concurrently
ctx := context.Background()
const numConcurrent = 10
done := make(chan bool, numConcurrent)
for i := 0; i < numConcurrent; i++ {
go func(idx int) {
callback(ctx, "127.0.0.1:6380", []string{string(rune(idx))})
done <- true
}(i)
}
// Wait for all to complete
for i := 0; i < numConcurrent; i++ {
<-done
}
// Verify all callbacks were invoked
if callbackCount.Load() != numConcurrent {
t.Errorf("Expected %d callback invocations, got %d", numConcurrent, callbackCount.Load())
}
t.Log("Concurrent callbacks test passed")
}
redis.go
+14
View File
@@ -27,7 +27,11 @@ const Nil = proto.Nil
// SetLogger set custom log
// Use with VoidLogger to disable logging.
// If logger is nil, the call is ignored and the existing logger is kept.
func SetLogger(logger internal.Logging) {
if logger == nil {
return
}
internal.Logger = logger
}
@@ -1130,6 +1134,16 @@ func (c *Client) Options() *Options {
return c.opt
}
// NodeAddress returns the address of the Redis node as reported by the server.
// For cluster clients, this is the endpoint from CLUSTER SLOTS before any transformation
// (e.g., loopback replacement). For standalone clients, this defaults to Addr.
//
// This is useful for matching the source field in maintenance notifications
// (e.g. SMIGRATED).
func (c *Client) NodeAddress() string {
return c.opt.NodeAddress
}
// GetMaintNotificationsManager returns the maintnotifications manager instance for monitoring and control.
// Returns nil if maintnotifications are not enabled.
func (c *Client) GetMaintNotificationsManager() *maintnotifications.Manager {
sentinel.go
+53 -30
View File
@@ -120,7 +120,12 @@ type FailoverOptions struct {
PoolFIFO bool
PoolSize int
PoolSize int
// MaxConcurrentDials is the maximum number of concurrent connection creation goroutines.
// If <= 0, defaults to PoolSize. If > PoolSize, it will be capped at PoolSize.
MaxConcurrentDials int
PoolTimeout time.Duration
MinIdleConns int
MaxIdleConns int
@@ -152,6 +157,10 @@ type FailoverOptions struct {
UnstableResp3 bool
// PushNotificationProcessor is the processor for handling push notifications.
// If nil, a default processor will be created for RESP3 connections.
PushNotificationProcessor push.NotificationProcessor
// MaintNotificationsConfig is not supported for FailoverClients at the moment
// MaintNotificationsConfig provides custom configuration for maintnotifications upgrades.
// When MaintNotificationsConfig.Mode is not "disabled", the client will handle
@@ -185,15 +194,17 @@ func (opt *FailoverOptions) clientOptions() *Options {
ReadBufferSize: opt.ReadBufferSize,
WriteBufferSize: opt.WriteBufferSize,
DialTimeout: opt.DialTimeout,
DialerRetries: opt.DialerRetries,
DialerRetryTimeout: opt.DialerRetryTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
DialTimeout: opt.DialTimeout,
DialerRetries: opt.DialerRetries,
DialerRetryTimeout: opt.DialerRetryTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
ContextTimeoutEnabled: opt.ContextTimeoutEnabled,
PoolFIFO: opt.PoolFIFO,
PoolSize: opt.PoolSize,
MaxConcurrentDials: opt.MaxConcurrentDials,
PoolTimeout: opt.PoolTimeout,
MinIdleConns: opt.MinIdleConns,
MaxIdleConns: opt.MaxIdleConns,
@@ -207,8 +218,9 @@ func (opt *FailoverOptions) clientOptions() *Options {
DisableIdentity: opt.DisableIdentity,
DisableIndentity: opt.DisableIndentity,
IdentitySuffix: opt.IdentitySuffix,
UnstableResp3: opt.UnstableResp3,
IdentitySuffix: opt.IdentitySuffix,
UnstableResp3: opt.UnstableResp3,
PushNotificationProcessor: opt.PushNotificationProcessor,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeDisabled,
@@ -236,15 +248,17 @@ func (opt *FailoverOptions) sentinelOptions(addr string) *Options {
ReadBufferSize: 4096,
WriteBufferSize: 4096,
DialTimeout: opt.DialTimeout,
DialerRetries: opt.DialerRetries,
DialerRetryTimeout: opt.DialerRetryTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
DialTimeout: opt.DialTimeout,
DialerRetries: opt.DialerRetries,
DialerRetryTimeout: opt.DialerRetryTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
ContextTimeoutEnabled: opt.ContextTimeoutEnabled,
PoolFIFO: opt.PoolFIFO,
PoolSize: opt.PoolSize,
MaxConcurrentDials: opt.MaxConcurrentDials,
PoolTimeout: opt.PoolTimeout,
MinIdleConns: opt.MinIdleConns,
MaxIdleConns: opt.MaxIdleConns,
@@ -258,8 +272,9 @@ func (opt *FailoverOptions) sentinelOptions(addr string) *Options {
DisableIdentity: opt.DisableIdentity,
DisableIndentity: opt.DisableIndentity,
IdentitySuffix: opt.IdentitySuffix,
UnstableResp3: opt.UnstableResp3,
IdentitySuffix: opt.IdentitySuffix,
UnstableResp3: opt.UnstableResp3,
PushNotificationProcessor: opt.PushNotificationProcessor,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeDisabled,
@@ -293,26 +308,31 @@ func (opt *FailoverOptions) clusterOptions() *ClusterOptions {
ReadBufferSize: opt.ReadBufferSize,
WriteBufferSize: opt.WriteBufferSize,
DialTimeout: opt.DialTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
DialTimeout: opt.DialTimeout,
DialerRetries: opt.DialerRetries,
DialerRetryTimeout: opt.DialerRetryTimeout,
ReadTimeout: opt.ReadTimeout,
WriteTimeout: opt.WriteTimeout,
ContextTimeoutEnabled: opt.ContextTimeoutEnabled,
PoolFIFO: opt.PoolFIFO,
PoolSize: opt.PoolSize,
PoolTimeout: opt.PoolTimeout,
MinIdleConns: opt.MinIdleConns,
MaxIdleConns: opt.MaxIdleConns,
MaxActiveConns: opt.MaxActiveConns,
ConnMaxIdleTime: opt.ConnMaxIdleTime,
ConnMaxLifetime: opt.ConnMaxLifetime,
PoolFIFO: opt.PoolFIFO,
PoolSize: opt.PoolSize,
MaxConcurrentDials: opt.MaxConcurrentDials,
PoolTimeout: opt.PoolTimeout,
MinIdleConns: opt.MinIdleConns,
MaxIdleConns: opt.MaxIdleConns,
MaxActiveConns: opt.MaxActiveConns,
ConnMaxIdleTime: opt.ConnMaxIdleTime,
ConnMaxLifetime: opt.ConnMaxLifetime,
TLSConfig: opt.TLSConfig,
DisableIdentity: opt.DisableIdentity,
DisableIndentity: opt.DisableIndentity,
IdentitySuffix: opt.IdentitySuffix,
FailingTimeoutSeconds: opt.FailingTimeoutSeconds,
DisableIdentity: opt.DisableIdentity,
DisableIndentity: opt.DisableIndentity,
IdentitySuffix: opt.IdentitySuffix,
FailingTimeoutSeconds: opt.FailingTimeoutSeconds,
PushNotificationProcessor: opt.PushNotificationProcessor,
MaintNotificationsConfig: &maintnotifications.Config{
Mode: maintnotifications.ModeDisabled,
@@ -416,11 +436,14 @@ func setupFailoverConnParams(u *url.URL, o *FailoverOptions) (*FailoverOptions,
o.MinRetryBackoff = q.duration("min_retry_backoff")
o.MaxRetryBackoff = q.duration("max_retry_backoff")
o.DialTimeout = q.duration("dial_timeout")
o.DialerRetries = q.int("dialer_retries")
o.DialerRetryTimeout = q.duration("dialer_retry_timeout")
o.ReadTimeout = q.duration("read_timeout")
o.WriteTimeout = q.duration("write_timeout")
o.ContextTimeoutEnabled = q.bool("context_timeout_enabled")
o.PoolFIFO = q.bool("pool_fifo")
o.PoolSize = q.int("pool_size")
o.MaxConcurrentDials = q.int("max_concurrent_dials")
o.MinIdleConns = q.int("min_idle_conns")
o.MaxIdleConns = q.int("max_idle_conns")
o.MaxActiveConns = q.int("max_active_conns")
universal.go
+72 -36
View File
@@ -8,6 +8,7 @@ import (
"github.com/redis/go-redis/v9/auth"
"github.com/redis/go-redis/v9/maintnotifications"
"github.com/redis/go-redis/v9/push"
)
// UniversalOptions information is required by UniversalClient to establish
@@ -57,7 +58,18 @@ type UniversalOptions struct {
MinRetryBackoff time.Duration
MaxRetryBackoff time.Duration
DialTimeout time.Duration
DialTimeout time.Duration
// DialerRetries is the maximum number of retry attempts when dialing fails.
//
// default: 5
DialerRetries int
// DialerRetryTimeout is the backoff duration between retry attempts.
//
// default: 100 milliseconds
DialerRetryTimeout time.Duration
ReadTimeout time.Duration
WriteTimeout time.Duration
ContextTimeoutEnabled bool
@@ -79,7 +91,12 @@ type UniversalOptions struct {
// PoolFIFO uses FIFO mode for each node connection pool GET/PUT (default LIFO).
PoolFIFO bool
PoolSize int
PoolSize int
// MaxConcurrentDials is the maximum number of concurrent connection creation goroutines.
// If <= 0, defaults to PoolSize. If > PoolSize, it will be capped at PoolSize.
MaxConcurrentDials int
PoolTimeout time.Duration
MinIdleConns int
MaxIdleConns int
@@ -122,6 +139,10 @@ type UniversalOptions struct {
UnstableResp3 bool
// PushNotificationProcessor is the processor for handling push notifications.
// If nil, a default processor will be created for RESP3 connections.
PushNotificationProcessor push.NotificationProcessor
// IsClusterMode can be used when only one Addrs is provided (e.g. Elasticache supports setting up cluster mode with configuration endpoint).
IsClusterMode bool
@@ -157,17 +178,20 @@ func (o *UniversalOptions) Cluster() *ClusterOptions {
MinRetryBackoff: o.MinRetryBackoff,
MaxRetryBackoff: o.MaxRetryBackoff,
DialTimeout: o.DialTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
DialTimeout: o.DialTimeout,
DialerRetries: o.DialerRetries,
DialerRetryTimeout: o.DialerRetryTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
ContextTimeoutEnabled: o.ContextTimeoutEnabled,
ReadBufferSize: o.ReadBufferSize,
WriteBufferSize: o.WriteBufferSize,
PoolFIFO: o.PoolFIFO,
PoolFIFO: o.PoolFIFO,
PoolSize: o.PoolSize,
MaxConcurrentDials: o.MaxConcurrentDials,
PoolTimeout: o.PoolTimeout,
MinIdleConns: o.MinIdleConns,
MaxIdleConns: o.MaxIdleConns,
@@ -178,12 +202,13 @@ func (o *UniversalOptions) Cluster() *ClusterOptions {
TLSConfig: o.TLSConfig,
DisableIdentity: o.DisableIdentity,
DisableIndentity: o.DisableIndentity,
IdentitySuffix: o.IdentitySuffix,
FailingTimeoutSeconds: o.FailingTimeoutSeconds,
UnstableResp3: o.UnstableResp3,
MaintNotificationsConfig: o.MaintNotificationsConfig,
DisableIdentity: o.DisableIdentity,
DisableIndentity: o.DisableIndentity,
IdentitySuffix: o.IdentitySuffix,
FailingTimeoutSeconds: o.FailingTimeoutSeconds,
UnstableResp3: o.UnstableResp3,
PushNotificationProcessor: o.PushNotificationProcessor,
MaintNotificationsConfig: o.MaintNotificationsConfig,
}
}
@@ -219,9 +244,12 @@ func (o *UniversalOptions) Failover() *FailoverOptions {
MinRetryBackoff: o.MinRetryBackoff,
MaxRetryBackoff: o.MaxRetryBackoff,
DialTimeout: o.DialTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
DialTimeout: o.DialTimeout,
DialerRetries: o.DialerRetries,
DialerRetryTimeout: o.DialerRetryTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
ContextTimeoutEnabled: o.ContextTimeoutEnabled,
ReadBufferSize: o.ReadBufferSize,
@@ -229,6 +257,7 @@ func (o *UniversalOptions) Failover() *FailoverOptions {
PoolFIFO: o.PoolFIFO,
PoolSize: o.PoolSize,
MaxConcurrentDials: o.MaxConcurrentDials,
PoolTimeout: o.PoolTimeout,
MinIdleConns: o.MinIdleConns,
MaxIdleConns: o.MaxIdleConns,
@@ -241,10 +270,11 @@ func (o *UniversalOptions) Failover() *FailoverOptions {
ReplicaOnly: o.ReadOnly,
DisableIdentity: o.DisableIdentity,
DisableIndentity: o.DisableIndentity,
IdentitySuffix: o.IdentitySuffix,
UnstableResp3: o.UnstableResp3,
DisableIdentity: o.DisableIdentity,
DisableIndentity: o.DisableIndentity,
IdentitySuffix: o.IdentitySuffix,
UnstableResp3: o.UnstableResp3,
PushNotificationProcessor: o.PushNotificationProcessor,
// Note: MaintNotificationsConfig not supported for FailoverOptions
}
}
@@ -274,30 +304,36 @@ func (o *UniversalOptions) Simple() *Options {
MinRetryBackoff: o.MinRetryBackoff,
MaxRetryBackoff: o.MaxRetryBackoff,
DialTimeout: o.DialTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
DialTimeout: o.DialTimeout,
DialerRetries: o.DialerRetries,
DialerRetryTimeout: o.DialerRetryTimeout,
ReadTimeout: o.ReadTimeout,
WriteTimeout: o.WriteTimeout,
ContextTimeoutEnabled: o.ContextTimeoutEnabled,
ReadBufferSize: o.ReadBufferSize,
WriteBufferSize: o.WriteBufferSize,
PoolFIFO: o.PoolFIFO,
PoolSize: o.PoolSize,
PoolTimeout: o.PoolTimeout,
MinIdleConns: o.MinIdleConns,
MaxIdleConns: o.MaxIdleConns,
MaxActiveConns: o.MaxActiveConns,
ConnMaxIdleTime: o.ConnMaxIdleTime,
ConnMaxLifetime: o.ConnMaxLifetime,
PoolFIFO: o.PoolFIFO,
PoolSize: o.PoolSize,
MaxConcurrentDials: o.MaxConcurrentDials,
PoolTimeout: o.PoolTimeout,
MinIdleConns: o.MinIdleConns,
MaxIdleConns: o.MaxIdleConns,
MaxActiveConns: o.MaxActiveConns,
ConnMaxIdleTime: o.ConnMaxIdleTime,
ConnMaxLifetime: o.ConnMaxLifetime,
ConnMaxLifetimeJitter: o.ConnMaxLifetimeJitter,
TLSConfig: o.TLSConfig,
DisableIdentity: o.DisableIdentity,
DisableIndentity: o.DisableIndentity,
IdentitySuffix: o.IdentitySuffix,
UnstableResp3: o.UnstableResp3,
MaintNotificationsConfig: o.MaintNotificationsConfig,
DisableIdentity: o.DisableIdentity,
DisableIndentity: o.DisableIndentity,
IdentitySuffix: o.IdentitySuffix,
UnstableResp3: o.UnstableResp3,
PushNotificationProcessor: o.PushNotificationProcessor,
MaintNotificationsConfig: o.MaintNotificationsConfig,
}
}