mirror of
https://github.com/hanzoai/redis.git
synced 2026-11-02 10:30:08 +00:00
Yuan WangandGitHub
a431c6c2e2
### Background AOF is often used as an effective data recovery method, but now if we have two AOFs from different nodes, it is hard to learn which one has latest data. Generally, we determine whose data is more up-to-date by reading the latest modification time of the AOF file, but because of replication delay, even if both master and replica write to the AOF at the same time, the data in the master is more up-to-date (there are commands that didn't arrive at the replica yet, or a large number of commands have accumulated on replica side ), so we may make wrong decision. ### Solution The replication offset always increments when AOF is enabled even if there is no replica, we think replication offset is better method to determine which one has more up-to-date data, whoever has a larger offset will have newer data, so we add the start replication offset info for AOF, as bellow. ``` file appendonly.aof.2.base.rdb seq 2 type b file appendonly.aof.2.incr.aof seq 2 type i startoffset 224 ``` And if we close gracefully the AOF file, not a crash, such as `shutdown`, `kill signal 15` or `config set appendonly no`, we will add the end replication offset, as bellow. ``` file appendonly.aof.2.base.rdb seq 2 type b file appendonly.aof.2.incr.aof seq 2 type i startoffset 224 endoffset 532 ``` #### Things to pay attention to - For BASE AOF, we do not add `startoffset` and `endoffset` info, since we could not know the start replication replication of data, and it is useless to help us to determine which one has more up-to-date data. - For AOFs from old version, we also don't add `startoffset` and `endoffset` info, since we also don't know start replication replication of them. If we add the start offset from 0, we might make the judgment even less accurate. For example, if the master has just rewritten the AOF, its INCR AOF will inevitably be very small. However, if the replica has not rewritten AOF for a long time, its INCR AOF might be much larger. By applying the following method, we might make incorrect decisions, so we still just check timestamp instead of adding offset info - If the last INCR AOF has `startoffset` or `endoffset`, we need to restore `server.master_repl_offset` according to them to avoid the rollback of the `startoffset` of next INCR AOF. If it has `endoffset`, we just use this value as `server.master_repl_offset`, and a very important thing is to remove this information from the manifest file to avoid the next time we load the manifest file with wrong `endoffset`. If it only has `startoffset`, we calculate `server.master_repl_offset` by the `startoffset` plus the file size. ### How to determine which one has more up-to-date data If AOF has a larger replication offset, it will have more up-to-date data. The following is how to get AOF offset: Read the AOF manifest file to obtain information about **the last INCR AOF** 1. If the last INCR AOF has `endoffset` field, we can directly use the `endoffset` to present the replication offset of AOF 2. If there is no `endoffset`(such as redis crashes abnormally), but there is `startoffset` filed of the last INCR AOF, we can get the replication offset of AOF by `startoffset` plus the file size 3. Finally, if the AOF doesn’t have both `startoffset` and `endoffset`, maybe from old version, and new version redis has not rewritten AOF yet, we still need to check the modification timestamp of the last INCR AOF ### TODO Fix ping causing inconsistency between AOF size and replication offset in the future PR. Because we increment the replication offset when sending PING/REPLCONF to the replica but do not write data to the AOF file, this might cause the starting offset of the AOF file plus its size to be inconsistent with the actual replication offset.
174 lines
4.3 KiB
Tcl
174 lines
4.3 KiB
Tcl
set ::base_aof_sufix ".base"
|
|
set ::incr_aof_sufix ".incr"
|
|
set ::manifest_suffix ".manifest"
|
|
set ::aof_format_suffix ".aof"
|
|
set ::rdb_format_suffix ".rdb"
|
|
|
|
proc get_full_path {dir filename} {
|
|
set _ [format "%s/%s" $dir $filename]
|
|
}
|
|
|
|
proc join_path {dir1 dir2} {
|
|
return [format "%s/%s" $dir1 $dir2]
|
|
}
|
|
|
|
proc get_redis_dir {} {
|
|
set config [srv config]
|
|
set _ [dict get $config "dir"]
|
|
}
|
|
|
|
proc check_file_exist {dir filename} {
|
|
set file_path [get_full_path $dir $filename]
|
|
return [file exists $file_path]
|
|
}
|
|
|
|
proc del_file {dir filename} {
|
|
set file_path [get_full_path $dir $filename]
|
|
catch {exec rm -rf $file_path}
|
|
}
|
|
|
|
proc get_cur_base_aof_name {manifest_filepath} {
|
|
set fp [open $manifest_filepath r+]
|
|
set lines {}
|
|
while {1} {
|
|
set line [gets $fp]
|
|
if {[eof $fp]} {
|
|
close $fp
|
|
break;
|
|
}
|
|
|
|
lappend lines $line
|
|
}
|
|
|
|
if {[llength $lines] == 0} {
|
|
return ""
|
|
}
|
|
|
|
set first_line [lindex $lines 0]
|
|
set aofname [lindex [split $first_line " "] 1]
|
|
set aoftype [lindex [split $first_line " "] 5]
|
|
if { $aoftype eq "b" } {
|
|
return $aofname
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
proc get_last_incr_aof_name {manifest_filepath} {
|
|
set fp [open $manifest_filepath r+]
|
|
set lines {}
|
|
while {1} {
|
|
set line [gets $fp]
|
|
if {[eof $fp]} {
|
|
close $fp
|
|
break;
|
|
}
|
|
|
|
lappend lines $line
|
|
}
|
|
|
|
if {[llength $lines] == 0} {
|
|
return ""
|
|
}
|
|
|
|
set len [llength $lines]
|
|
set last_line [lindex $lines [expr $len - 1]]
|
|
set aofname [lindex [split $last_line " "] 1]
|
|
set aoftype [lindex [split $last_line " "] 5]
|
|
if { $aoftype eq "i" } {
|
|
return $aofname
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
proc get_last_incr_aof_path {r} {
|
|
set dir [lindex [$r config get dir] 1]
|
|
set appenddirname [lindex [$r config get appenddirname] 1]
|
|
set appendfilename [lindex [$r config get appendfilename] 1]
|
|
set manifest_filepath [file join $dir $appenddirname $appendfilename$::manifest_suffix]
|
|
set last_incr_aof_name [get_last_incr_aof_name $manifest_filepath]
|
|
if {$last_incr_aof_name == ""} {
|
|
return ""
|
|
}
|
|
return [file join $dir $appenddirname $last_incr_aof_name]
|
|
}
|
|
|
|
proc get_base_aof_path {r} {
|
|
set dir [lindex [$r config get dir] 1]
|
|
set appenddirname [lindex [$r config get appenddirname] 1]
|
|
set appendfilename [lindex [$r config get appendfilename] 1]
|
|
set manifest_filepath [file join $dir $appenddirname $appendfilename$::manifest_suffix]
|
|
set cur_base_aof_name [get_cur_base_aof_name $manifest_filepath]
|
|
if {$cur_base_aof_name == ""} {
|
|
return ""
|
|
}
|
|
return [file join $dir $appenddirname $cur_base_aof_name]
|
|
}
|
|
|
|
proc assert_aof_manifest_content {manifest_path content} {
|
|
set fp [open $manifest_path r+]
|
|
set lines {}
|
|
while {1} {
|
|
set line [gets $fp]
|
|
if {[eof $fp]} {
|
|
close $fp
|
|
break;
|
|
}
|
|
|
|
lappend lines $line
|
|
}
|
|
|
|
assert_equal [llength $lines] [llength $content]
|
|
|
|
for { set i 0 } { $i < [llength $lines] } {incr i} {
|
|
assert {[string first [lindex $content $i] [lindex $lines $i]] != -1}
|
|
}
|
|
}
|
|
|
|
proc clean_aof_persistence {aof_dirpath} {
|
|
catch {eval exec rm -rf [glob $aof_dirpath]}
|
|
}
|
|
|
|
proc append_to_manifest {str} {
|
|
upvar fp fp
|
|
puts -nonewline $fp $str
|
|
}
|
|
|
|
proc create_aof_manifest {dir aof_manifest_file code} {
|
|
create_aof_dir $dir
|
|
upvar fp fp
|
|
set fp [open $aof_manifest_file w+]
|
|
uplevel 1 $code
|
|
close $fp
|
|
}
|
|
|
|
proc append_to_aof {str} {
|
|
upvar fp fp
|
|
puts -nonewline $fp $str
|
|
}
|
|
|
|
proc create_aof {dir aof_file code} {
|
|
create_aof_dir $dir
|
|
upvar fp fp
|
|
set fp [open $aof_file w+]
|
|
uplevel 1 $code
|
|
close $fp
|
|
}
|
|
|
|
proc create_aof_dir {dir_path} {
|
|
file mkdir $dir_path
|
|
}
|
|
|
|
proc start_server_aof {overrides code} {
|
|
upvar defaults defaults srv srv server_path server_path aof_basename aof_basename aof_dirpath aof_dirpath aof_manifest_file aof_manifest_file aof_manifest_file2 aof_manifest_file2
|
|
set config [concat $defaults $overrides]
|
|
start_server [list overrides $config keep_persistence true] $code
|
|
}
|
|
|
|
proc start_server_aof_ex {overrides options code} {
|
|
upvar defaults defaults srv srv server_path server_path
|
|
set config [concat $defaults $overrides]
|
|
start_server [concat [list overrides $config keep_persistence true] $options] $code
|
|
}
|