mirror of
https://github.com/hanzoai/kv.git
synced 2026-11-09 16:05:35 +00:00
Previously, changing the cluster-require-full-coverage config via
CONFIG SET would not trigger an immediate cluster state update, requiring
the cluster to wait for the next clusterCron cycle to reflect the change.
Furthermore, there was a bug when flipping cluster-require-full-coverage
from "no" to "yes" while the cluster was in the OK state. Because
clusterCron only called clusterUpdateState() when:
```
if (update_state || server.cluster->state == CLUSTER_FAIL)
```
The cluster would remain stuck in the OK state despite having uncovered
slots, since neither condition was true (state was OK and update_state
was not set).
The fix schedules CLUSTER_TODO_UPDATE_STATE in beforeSleep whenever
the config changes, so the state is re-evaluated promptly in both
directions (FAIL -> OK and OK -> FAIL).
Add a test that verifies both transitions by pausing a primary with
cluster-require-full-coverage set to "no" (cluster stays OK), then
flipping it back to "yes" (cluster immediately transitions to FAIL).
Signed-off-by: Binbin <[email protected]>
138 lines
5.8 KiB
Tcl
138 lines
5.8 KiB
Tcl
start_cluster 2 2 {tags {external:skip cluster}} {
|
|
test {Key lazy expires during key migration} {
|
|
R 0 DEBUG SET-ACTIVE-EXPIRE 0
|
|
|
|
set key_slot [R 0 CLUSTER KEYSLOT FOO]
|
|
R 0 set FOO BAR PX 10
|
|
set src_id [R 0 CLUSTER MYID]
|
|
set trg_id [R 1 CLUSTER MYID]
|
|
R 0 CLUSTER SETSLOT $key_slot MIGRATING $trg_id
|
|
R 1 CLUSTER SETSLOT $key_slot IMPORTING $src_id
|
|
after 11
|
|
assert_error {ASK*} {R 0 GET FOO}
|
|
R 0 ping
|
|
} {PONG}
|
|
|
|
test "Coverage: Basic cluster commands" {
|
|
assert_equal {OK} [R 0 CLUSTER saveconfig]
|
|
|
|
set id [R 0 CLUSTER MYID]
|
|
assert_equal {0} [R 0 CLUSTER count-failure-reports $id]
|
|
|
|
R 0 flushall
|
|
assert_equal {OK} [R 0 CLUSTER flushslots]
|
|
}
|
|
}
|
|
|
|
start_cluster 1 1 {tags {external:skip cluster}} {
|
|
test {Cross-slot transaction} {
|
|
assert_equal OK [R 0 multi]
|
|
assert_equal QUEUED [r get foo]
|
|
assert_equal QUEUED [r get bar]
|
|
assert_error {CROSSSLOT *} {r exec}
|
|
}
|
|
}
|
|
|
|
# Create a folder called "nodes.conf" to trigger temp nodes.conf rename
|
|
# failure and it will cause cluster config file save to fail at the rename.
|
|
proc create_nodes_conf_folder {srv_idx} {
|
|
set dir [lindex [R $srv_idx config get dir] 1]
|
|
set cluster_conf [lindex [R $srv_idx config get cluster-config-file] 1]
|
|
set cluster_conf_path [file join $dir $cluster_conf]
|
|
if {[file exists $cluster_conf_path]} { exec rm -f $cluster_conf_path }
|
|
exec mkdir -p $cluster_conf_path
|
|
}
|
|
|
|
start_cluster 1 1 {tags {external:skip cluster} overrides {cluster-config-save-behavior sync}} {
|
|
test {cluster-config-save-behavior sync mode - node exits when config save fails} {
|
|
# Create folder that can cause the rename fail.
|
|
create_nodes_conf_folder 1
|
|
|
|
# Trigger a takeover so that cluster will need to update the config file.
|
|
catch {R 1 cluster failover takeover}
|
|
|
|
# Wait for R1 to exit due to config save failure.
|
|
wait_for_condition 1000 50 {
|
|
[process_is_alive [srv -1 pid]] == 0
|
|
} else {
|
|
fail "R1 did not exit"
|
|
}
|
|
|
|
# Verify that save failure and fatal exit logs were printed.
|
|
verify_log_message -1 "*Could not rename tmp cluster config file*" 0
|
|
verify_log_message -1 "*Fatal: can't update cluster config file*" 0
|
|
}
|
|
}
|
|
|
|
start_cluster 1 1 {tags {external:skip cluster} overrides {cluster-config-save-behavior best-effort}} {
|
|
test {cluster-config-save-behavior best-effort mode - node continues running when config save fails} {
|
|
# Create folder that can cause the rename fail.
|
|
create_nodes_conf_folder 0
|
|
create_nodes_conf_folder 1
|
|
|
|
# Trigger a takeover so that cluster will need to update the config file.
|
|
R 1 cluster failover takeover
|
|
wait_for_condition 1000 50 {
|
|
[s 0 role] eq {slave} &&
|
|
[s -1 role] eq {master}
|
|
} else {
|
|
fail "The failover does not happen"
|
|
}
|
|
|
|
# Make sure the process is still alive, we won't exit when fail to save the config file.
|
|
assert_equal {PONG} [R 0 ping]
|
|
assert_equal {PONG} [R 1 ping]
|
|
assert_equal 1 [process_is_alive [srv 0 pid]]
|
|
assert_equal 1 [process_is_alive [srv -1 pid]]
|
|
|
|
# Make sure relevant logs are printed.
|
|
verify_log_message 0 "*Could not rename tmp cluster config file*" 0
|
|
verify_log_message -1 "*Could not rename tmp cluster config file*" 0
|
|
verify_log_message 0 "*Cluster config updated even though writing the cluster config file to disk failed*" 0
|
|
verify_log_message -1 "*Cluster config updated even though writing the cluster config file to disk failed*" 0
|
|
|
|
# Trigger a takeover so that cluster will need to update the config file.
|
|
# We will not frequently print the "save failed" log.
|
|
R 0 cluster failover takeover
|
|
wait_for_condition 1000 50 {
|
|
[s 0 role] eq {master} &&
|
|
[s -1 role] eq {slave}
|
|
} else {
|
|
fail "The failover does not happen"
|
|
}
|
|
assert_morethan_equal [count_log_message 0 "Could not rename tmp cluster config file"] 2
|
|
assert_equal [count_log_message 0 "Cluster config updated even though writing the cluster config file to disk failed"] 1
|
|
assert_morethan_equal [count_log_message -1 "Could not rename tmp cluster config file"] 2
|
|
assert_equal [count_log_message -1 "Cluster config updated even though writing the cluster config file to disk failed"] 1
|
|
}
|
|
}
|
|
|
|
# Test that changing cluster-require-full-coverage triggers an immediate
|
|
# cluster state update. Use 3 primaries with no replicas so that pausing
|
|
# one primary leaves a slot range uncovered without automatic failover.
|
|
start_cluster 3 0 {tags {external:skip cluster} overrides {cluster-require-full-coverage yes}} {
|
|
test "Cluster state transitions when toggling cluster-require-full-coverage" {
|
|
# Pause one primary so part of the slots is uncovered.
|
|
pause_process [srv 0 pid]
|
|
wait_for_cluster_state fail
|
|
|
|
# With cluster-require-full-coverage set to no, the R1 should
|
|
# stay in the OK state even though some slots are uncovered.
|
|
R 1 config set cluster-require-full-coverage no
|
|
assert_equal [CI 1 cluster_state] "ok"
|
|
assert_equal [CI 2 cluster_state] "fail"
|
|
|
|
# Now change cluster-require-full-coverage back to yes. The R1
|
|
# should immediately transition to FAIL because slots are uncovered.
|
|
# Without the fix, the cluster would remain stuck in OK because
|
|
# clusterUpdateState was only called when cluster is FAIL in clusterCron.
|
|
R 1 config set cluster-require-full-coverage yes
|
|
assert_equal [CI 1 cluster_state] "fail"
|
|
assert_equal [CI 2 cluster_state] "fail"
|
|
|
|
# Resume the paused primary to bring the cluster back to OK.
|
|
resume_process [srv 0 pid]
|
|
wait_for_cluster_state ok
|
|
}
|
|
}
|