mirror of
https://github.com/hanzoai/redis.git
synced 2026-10-30 18:25:12 +00:00
Yuan WangandGitHub
a431c6c2e2
### Background AOF is often used as an effective data recovery method, but now if we have two AOFs from different nodes, it is hard to learn which one has latest data. Generally, we determine whose data is more up-to-date by reading the latest modification time of the AOF file, but because of replication delay, even if both master and replica write to the AOF at the same time, the data in the master is more up-to-date (there are commands that didn't arrive at the replica yet, or a large number of commands have accumulated on replica side ), so we may make wrong decision. ### Solution The replication offset always increments when AOF is enabled even if there is no replica, we think replication offset is better method to determine which one has more up-to-date data, whoever has a larger offset will have newer data, so we add the start replication offset info for AOF, as bellow. ``` file appendonly.aof.2.base.rdb seq 2 type b file appendonly.aof.2.incr.aof seq 2 type i startoffset 224 ``` And if we close gracefully the AOF file, not a crash, such as `shutdown`, `kill signal 15` or `config set appendonly no`, we will add the end replication offset, as bellow. ``` file appendonly.aof.2.base.rdb seq 2 type b file appendonly.aof.2.incr.aof seq 2 type i startoffset 224 endoffset 532 ``` #### Things to pay attention to - For BASE AOF, we do not add `startoffset` and `endoffset` info, since we could not know the start replication replication of data, and it is useless to help us to determine which one has more up-to-date data. - For AOFs from old version, we also don't add `startoffset` and `endoffset` info, since we also don't know start replication replication of them. If we add the start offset from 0, we might make the judgment even less accurate. For example, if the master has just rewritten the AOF, its INCR AOF will inevitably be very small. However, if the replica has not rewritten AOF for a long time, its INCR AOF might be much larger. By applying the following method, we might make incorrect decisions, so we still just check timestamp instead of adding offset info - If the last INCR AOF has `startoffset` or `endoffset`, we need to restore `server.master_repl_offset` according to them to avoid the rollback of the `startoffset` of next INCR AOF. If it has `endoffset`, we just use this value as `server.master_repl_offset`, and a very important thing is to remove this information from the manifest file to avoid the next time we load the manifest file with wrong `endoffset`. If it only has `startoffset`, we calculate `server.master_repl_offset` by the `startoffset` plus the file size. ### How to determine which one has more up-to-date data If AOF has a larger replication offset, it will have more up-to-date data. The following is how to get AOF offset: Read the AOF manifest file to obtain information about **the last INCR AOF** 1. If the last INCR AOF has `endoffset` field, we can directly use the `endoffset` to present the replication offset of AOF 2. If there is no `endoffset`(such as redis crashes abnormally), but there is `startoffset` filed of the last INCR AOF, we can get the replication offset of AOF by `startoffset` plus the file size 3. Finally, if the AOF doesn’t have both `startoffset` and `endoffset`, maybe from old version, and new version redis has not rewritten AOF yet, we still need to check the modification timestamp of the last INCR AOF ### TODO Fix ping causing inconsistency between AOF size and replication offset in the future PR. Because we increment the replication offset when sending PING/REPLCONF to the replica but do not write data to the AOF file, this might cause the starting offset of the AOF file plus its size to be inconsistent with the actual replication offset.
1539 lines
60 KiB
Tcl
1539 lines
60 KiB
Tcl
source tests/support/aofmanifest.tcl
|
|
set defaults {appendonly {yes} appendfilename {appendonly.aof} appenddirname {appendonlydir} auto-aof-rewrite-percentage {0}}
|
|
set server_path [tmpdir server.multi.aof]
|
|
set aof_dirname "appendonlydir"
|
|
set aof_basename "appendonly.aof"
|
|
set aof_dirpath "$server_path/$aof_dirname"
|
|
set aof_base1_file "$server_path/$aof_dirname/${aof_basename}.1$::base_aof_sufix$::aof_format_suffix"
|
|
set aof_base2_file "$server_path/$aof_dirname/${aof_basename}.2$::base_aof_sufix$::aof_format_suffix"
|
|
set aof_incr1_file "$server_path/$aof_dirname/${aof_basename}.1$::incr_aof_sufix$::aof_format_suffix"
|
|
set aof_incr2_file "$server_path/$aof_dirname/${aof_basename}.2$::incr_aof_sufix$::aof_format_suffix"
|
|
set aof_incr3_file "$server_path/$aof_dirname/${aof_basename}.3$::incr_aof_sufix$::aof_format_suffix"
|
|
set aof_manifest_file "$server_path/$aof_dirname/${aof_basename}$::manifest_suffix"
|
|
set aof_old_name_old_path "$server_path/$aof_basename"
|
|
set aof_old_name_new_path "$aof_dirpath/$aof_basename"
|
|
set aof_old_name_old_path2 "$server_path/${aof_basename}2"
|
|
set aof_manifest_file2 "$server_path/$aof_dirname/${aof_basename}2$::manifest_suffix"
|
|
|
|
tags {"external:skip"} {
|
|
|
|
# Test Part 1
|
|
|
|
# In order to test the loading logic of redis under different combinations of manifest and AOF.
|
|
# We will manually construct the manifest file and AOF, and then start redis to verify whether
|
|
# the redis behavior is as expected.
|
|
|
|
test {Multi Part AOF can't load data when some file missing} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr2_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
append_to_manifest "file appendonly.aof.2.incr.aof seq 2 type i\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 1 [count_message_lines $server_path/stdout "appendonly.aof.1.incr.aof .*No such file or directory"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when the sequence not increase monotonically} {
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr2_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.2.incr.aof seq 2 type i\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 1 [count_message_lines $server_path/stdout "Found a non-monotonic sequence number"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when there are blank lines in the manifest file} {
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr3_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
append_to_manifest "\n"
|
|
append_to_manifest "file appendonly.aof.3.incr.aof seq 3 type i\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 1 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when there is a duplicate base file} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_base2_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.2.base.aof seq 2 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 1 [count_message_lines $server_path/stdout "Found duplicate base file information"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when the manifest format is wrong (type unknown)} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type x\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 1 [count_message_lines $server_path/stdout "Unknown AOF file type"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when the manifest format is wrong (missing key)} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "filx appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 2 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when the manifest format is wrong (line too short)} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof type i\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 3 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when the manifest format is wrong (line too long)} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 1 [count_message_lines $server_path/stdout "The AOF manifest file contains too long line"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when the manifest format is wrong (odd parameter)} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i newkey\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 4 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when the manifest file is empty} {
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 1 [count_message_lines $server_path/stdout "Found an empty AOF manifest"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can start when no aof and no manifest} {
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
|
|
assert_equal OK [$client set k1 v1]
|
|
assert_equal v1 [$client get k1]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can start when we have en empty AOF dir} {
|
|
create_aof_dir $aof_dirpath
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
}
|
|
}
|
|
|
|
test {Multi Part AOF can load data discontinuously increasing sequence} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr3_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
append_to_manifest "file appendonly.aof.3.incr.aof seq 3 type i\n"
|
|
}
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal v1 [$client get k1]
|
|
assert_equal v2 [$client get k2]
|
|
assert_equal v3 [$client get k3]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can load data when manifest add new k-v} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr3_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b newkey newvalue\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
append_to_manifest "file appendonly.aof.3.incr.aof seq 3 type i\n"
|
|
}
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal v1 [$client get k1]
|
|
assert_equal v2 [$client get k2]
|
|
assert_equal v3 [$client get k3]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can load data when some AOFs are empty} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr3_file {
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
append_to_manifest "file appendonly.aof.3.incr.aof seq 3 type i\n"
|
|
}
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal v1 [$client get k1]
|
|
assert_equal "" [$client get k2]
|
|
assert_equal v3 [$client get k3]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can load data from old version redis (rdb preamble no)} {
|
|
create_aof $server_path $aof_old_name_old_path {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
append_to_aof [formatCommand set k2 v2]
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal v1 [$client get k1]
|
|
assert_equal v2 [$client get k2]
|
|
assert_equal v3 [$client get k3]
|
|
|
|
assert_equal 0 [check_file_exist $server_path $aof_basename]
|
|
assert_equal 1 [check_file_exist $aof_dirpath $aof_basename]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
assert_equal OK [$client set k4 v4]
|
|
|
|
$client bgrewriteaof
|
|
waitForBgrewriteaof $client
|
|
|
|
assert_equal OK [$client set k5 v5]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.2.base.rdb seq 2 type b}
|
|
{file appendonly.aof.2.incr.aof seq 2 type i}
|
|
}
|
|
|
|
set d1 [$client debug digest]
|
|
$client debug loadaof
|
|
set d2 [$client debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can load data from old version redis (rdb preamble yes)} {
|
|
exec cp tests/assets/rdb-preamble.aof $aof_old_name_old_path
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
# k1 k2 in rdb header and k3 in AOF tail
|
|
assert_equal v1 [$client get k1]
|
|
assert_equal v2 [$client get k2]
|
|
assert_equal v3 [$client get k3]
|
|
|
|
assert_equal 0 [check_file_exist $server_path $aof_basename]
|
|
assert_equal 1 [check_file_exist $aof_dirpath $aof_basename]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
assert_equal OK [$client set k4 v4]
|
|
|
|
$client bgrewriteaof
|
|
waitForBgrewriteaof $client
|
|
|
|
assert_equal OK [$client set k5 v5]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.2.base.rdb seq 2 type b}
|
|
{file appendonly.aof.2.incr.aof seq 2 type i}
|
|
}
|
|
|
|
set d1 [$client debug digest]
|
|
$client debug loadaof
|
|
set d2 [$client debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can continue the upgrade from the interrupted upgrade state} {
|
|
create_aof $server_path $aof_old_name_old_path {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
append_to_aof [formatCommand set k2 v2]
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
# Create a layout of an interrupted upgrade (interrupted before the rename).
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof seq 1 type b\n"
|
|
}
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal v1 [$client get k1]
|
|
assert_equal v2 [$client get k2]
|
|
assert_equal v3 [$client get k3]
|
|
|
|
assert_equal 0 [check_file_exist $server_path $aof_basename]
|
|
assert_equal 1 [check_file_exist $aof_dirpath $aof_basename]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can be loaded correctly when both server dir and aof dir contain old AOF} {
|
|
create_aof $server_path $aof_old_name_old_path {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
append_to_aof [formatCommand set k2 v2]
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof seq 1 type b\n"
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_old_name_new_path {
|
|
append_to_aof [formatCommand set k4 v4]
|
|
append_to_aof [formatCommand set k5 v5]
|
|
append_to_aof [formatCommand set k6 v6]
|
|
}
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal 0 [$client exists k1]
|
|
assert_equal 0 [$client exists k2]
|
|
assert_equal 0 [$client exists k3]
|
|
|
|
assert_equal v4 [$client get k4]
|
|
assert_equal v5 [$client get k5]
|
|
assert_equal v6 [$client get k6]
|
|
|
|
assert_equal 1 [check_file_exist $server_path $aof_basename]
|
|
assert_equal 1 [check_file_exist $aof_dirpath $aof_basename]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
catch {exec rm -rf $aof_old_name_old_path}
|
|
}
|
|
|
|
test {Multi Part AOF can't load data when the manifest contains the old AOF file name but the file does not exist in server dir and aof dir} {
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof seq 1 type b\n"
|
|
}
|
|
|
|
start_server_aof_ex [list dir $server_path] [list wait_ready false] {
|
|
wait_for_condition 100 50 {
|
|
! [is_alive [srv pid]]
|
|
} else {
|
|
fail "AOF loading didn't fail"
|
|
}
|
|
|
|
assert_equal 1 [count_message_lines $server_path/stdout "appendonly.aof .*No such file or directory"]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can upgrade when when two redis share the same server dir} {
|
|
create_aof $server_path $aof_old_name_old_path {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
append_to_aof [formatCommand set k2 v2]
|
|
append_to_aof [formatCommand set k3 v3]
|
|
}
|
|
|
|
create_aof $server_path $aof_old_name_old_path2 {
|
|
append_to_aof [formatCommand set k4 v4]
|
|
append_to_aof [formatCommand set k5 v5]
|
|
append_to_aof [formatCommand set k6 v6]
|
|
}
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
set redis1 [redis [srv host] [srv port] 0 $::tls]
|
|
|
|
start_server [list overrides [list dir $server_path appendonly yes appendfilename appendonly.aof2]] {
|
|
set redis2 [redis [srv host] [srv port] 0 $::tls]
|
|
|
|
test "Multi Part AOF can upgrade when when two redis share the same server dir (redis1)" {
|
|
wait_done_loading $redis1
|
|
assert_equal v1 [$redis1 get k1]
|
|
assert_equal v2 [$redis1 get k2]
|
|
assert_equal v3 [$redis1 get k3]
|
|
|
|
assert_equal 0 [$redis1 exists k4]
|
|
assert_equal 0 [$redis1 exists k5]
|
|
assert_equal 0 [$redis1 exists k6]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
$redis1 bgrewriteaof
|
|
waitForBgrewriteaof $redis1
|
|
|
|
assert_equal OK [$redis1 set k v]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.2.base.rdb seq 2 type b}
|
|
{file appendonly.aof.2.incr.aof seq 2 type i}
|
|
}
|
|
|
|
set d1 [$redis1 debug digest]
|
|
$redis1 debug loadaof
|
|
set d2 [$redis1 debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
test "Multi Part AOF can upgrade when when two redis share the same server dir (redis2)" {
|
|
wait_done_loading $redis2
|
|
|
|
assert_equal 0 [$redis2 exists k1]
|
|
assert_equal 0 [$redis2 exists k2]
|
|
assert_equal 0 [$redis2 exists k3]
|
|
|
|
assert_equal v4 [$redis2 get k4]
|
|
assert_equal v5 [$redis2 get k5]
|
|
assert_equal v6 [$redis2 get k6]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file2 {
|
|
{file appendonly.aof2 seq 1 type b}
|
|
{file appendonly.aof2.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
$redis2 bgrewriteaof
|
|
waitForBgrewriteaof $redis2
|
|
|
|
assert_equal OK [$redis2 set k v]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file2 {
|
|
{file appendonly.aof2.2.base.rdb seq 2 type b}
|
|
{file appendonly.aof2.2.incr.aof seq 2 type i}
|
|
}
|
|
|
|
set d1 [$redis2 debug digest]
|
|
$redis2 debug loadaof
|
|
set d2 [$redis2 debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
test {Multi Part AOF can handle appendfilename contains whitespaces} {
|
|
start_server [list overrides [list appendonly yes appendfilename "\" file seq \\n\\n.aof \""]] {
|
|
set dir [get_redis_dir]
|
|
set aof_manifest_name [format "%s/%s/%s%s" $dir "appendonlydir" " file seq \n\n.aof " $::manifest_suffix]
|
|
set redis [redis [srv host] [srv port] 0 $::tls]
|
|
|
|
assert_equal OK [$redis set k1 v1]
|
|
|
|
$redis bgrewriteaof
|
|
waitForBgrewriteaof $redis
|
|
|
|
assert_aof_manifest_content $aof_manifest_name {
|
|
{file " file seq \n\n.aof .2.base.rdb" seq 2 type b}
|
|
{file " file seq \n\n.aof .2.incr.aof" seq 2 type i}
|
|
}
|
|
|
|
set d1 [$redis debug digest]
|
|
$redis debug loadaof
|
|
set d2 [$redis debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can create BASE (RDB format) when redis starts from empty} {
|
|
start_server_aof [list dir $server_path] {
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.rdb seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
$client set foo behavior
|
|
|
|
set d1 [$client debug digest]
|
|
$client debug loadaof
|
|
set d2 [$client debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can create BASE (AOF format) when redis starts from empty} {
|
|
start_server_aof [list dir $server_path aof-use-rdb-preamble no] {
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::aof_format_suffix}"]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.aof seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
$client set foo behavior
|
|
|
|
set d1 [$client debug digest]
|
|
$client debug loadaof
|
|
set d2 [$client debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
# Test Part 2
|
|
#
|
|
# To test whether the AOFRW behaves as expected during the redis run.
|
|
# We will start redis first, then perform pressure writing, enable and disable AOF, and manually
|
|
# and automatically run bgrewrite and other actions, to test whether the correct AOF file is created,
|
|
# whether the correct manifest is generated, whether the data can be reload correctly under continuous
|
|
# writing pressure, etc.
|
|
|
|
|
|
start_server {tags {"Multi Part AOF"} overrides {aof-use-rdb-preamble {yes} appendonly {no} save {}}} {
|
|
set dir [get_redis_dir]
|
|
set aof_basename "appendonly.aof"
|
|
set aof_dirname "appendonlydir"
|
|
set aof_dirpath "$dir/$aof_dirname"
|
|
set aof_manifest_name "$aof_basename$::manifest_suffix"
|
|
set aof_manifest_file "$dir/$aof_dirname/$aof_manifest_name"
|
|
|
|
set master [srv 0 client]
|
|
set master_host [srv 0 host]
|
|
set master_port [srv 0 port]
|
|
|
|
catch {exec rm -rf $aof_manifest_file}
|
|
|
|
test "Make sure aof manifest $aof_manifest_name not in aof directory" {
|
|
assert_equal 0 [file exists $aof_manifest_file]
|
|
}
|
|
|
|
test "AOF enable will create manifest file" {
|
|
r config set appendonly yes ; # Will create manifest and new INCR aof
|
|
r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite.
|
|
waitForBgrewriteaof r
|
|
|
|
# Start write load
|
|
set load_handle0 [start_write_load $master_host $master_port 10]
|
|
|
|
wait_for_condition 50 100 {
|
|
[r dbsize] > 0
|
|
} else {
|
|
fail "No write load detected."
|
|
}
|
|
|
|
# First AOFRW done
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.rdb seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
# Check we really have these files
|
|
assert_equal 1 [check_file_exist $aof_dirpath $aof_manifest_name]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
|
|
r bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
|
|
# The second AOFRW done
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.2.base.rdb seq 2 type b}
|
|
{file appendonly.aof.2.incr.aof seq 2 type i}
|
|
}
|
|
|
|
assert_equal 1 [check_file_exist $aof_dirpath $aof_manifest_name]
|
|
# Wait bio delete history
|
|
wait_for_condition 1000 10 {
|
|
[check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::rdb_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"] == 0
|
|
} else {
|
|
fail "Failed to delete history AOF"
|
|
}
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
|
|
stop_write_load $load_handle0
|
|
wait_load_handlers_disconnected
|
|
|
|
set d1 [r debug digest]
|
|
r debug loadaof
|
|
set d2 [r debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
test "AOF multiple rewrite failures will open multiple INCR AOFs" {
|
|
# Start write load
|
|
r config set rdb-key-save-delay 10000000
|
|
|
|
set orig_size [r dbsize]
|
|
set load_handle0 [start_write_load $master_host $master_port 10]
|
|
|
|
wait_for_condition 50 100 {
|
|
[r dbsize] > $orig_size
|
|
} else {
|
|
fail "No write load detected."
|
|
}
|
|
|
|
# Let AOFRW fail three times
|
|
r bgrewriteaof
|
|
set pid1 [get_child_pid 0]
|
|
catch {exec kill -9 $pid1}
|
|
waitForBgrewriteaof r
|
|
|
|
r bgrewriteaof
|
|
set pid2 [get_child_pid 0]
|
|
catch {exec kill -9 $pid2}
|
|
waitForBgrewriteaof r
|
|
|
|
r bgrewriteaof
|
|
set pid3 [get_child_pid 0]
|
|
catch {exec kill -9 $pid3}
|
|
waitForBgrewriteaof r
|
|
|
|
assert_equal 0 [check_file_exist $dir "temp-rewriteaof-bg-$pid1.aof"]
|
|
assert_equal 0 [check_file_exist $dir "temp-rewriteaof-bg-$pid2.aof"]
|
|
assert_equal 0 [check_file_exist $dir "temp-rewriteaof-bg-$pid3.aof"]
|
|
|
|
# We will have four INCR AOFs
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.2.base.rdb seq 2 type b}
|
|
{file appendonly.aof.2.incr.aof seq 2 type i}
|
|
{file appendonly.aof.3.incr.aof seq 3 type i}
|
|
{file appendonly.aof.4.incr.aof seq 4 type i}
|
|
{file appendonly.aof.5.incr.aof seq 5 type i}
|
|
}
|
|
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.3${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.4${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.5${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
|
|
stop_write_load $load_handle0
|
|
wait_load_handlers_disconnected
|
|
|
|
set d1 [r debug digest]
|
|
r debug loadaof
|
|
set d2 [r debug digest]
|
|
assert {$d1 eq $d2}
|
|
|
|
r config set rdb-key-save-delay 0
|
|
catch {exec kill -9 [get_child_pid 0]}
|
|
wait_for_condition 1000 10 {
|
|
[s rdb_bgsave_in_progress] eq 0
|
|
} else {
|
|
fail "bgsave did not stop in time"
|
|
}
|
|
|
|
# AOFRW success
|
|
r bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
|
|
# All previous INCR AOFs have become history
|
|
# and have be deleted
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.3.base.rdb seq 3 type b}
|
|
{file appendonly.aof.6.incr.aof seq 6 type i}
|
|
}
|
|
|
|
# Wait bio delete history
|
|
wait_for_condition 1000 10 {
|
|
[check_file_exist $aof_dirpath "${aof_basename}.2${::base_aof_sufix}${::rdb_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.3${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.4${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.5${::incr_aof_sufix}${::aof_format_suffix}"] == 0
|
|
} else {
|
|
fail "Failed to delete history AOF"
|
|
}
|
|
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.3${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.6${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
|
|
set d1 [r debug digest]
|
|
r debug loadaof
|
|
set d2 [r debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
test "AOF rewrite doesn't open new aof when AOF turn off" {
|
|
r config set appendonly no
|
|
|
|
r bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
|
|
# We only have BASE AOF, no INCR AOF
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.4.base.rdb seq 4 type b}
|
|
}
|
|
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.4${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
wait_for_condition 1000 10 {
|
|
[check_file_exist $aof_dirpath "${aof_basename}.6${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.7${::incr_aof_sufix}${::aof_format_suffix}"] == 0
|
|
} else {
|
|
fail "Failed to delete history AOF"
|
|
}
|
|
|
|
set d1 [r debug digest]
|
|
r debug loadaof
|
|
set d2 [r debug digest]
|
|
assert {$d1 eq $d2}
|
|
|
|
# Turn on AOF again
|
|
r config set appendonly yes
|
|
waitForBgrewriteaof r
|
|
|
|
# A new INCR AOF was created
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.5.base.rdb seq 5 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
# Wait bio delete history
|
|
wait_for_condition 1000 10 {
|
|
[check_file_exist $aof_dirpath "${aof_basename}.4${::base_aof_sufix}${::rdb_format_suffix}"] == 0
|
|
} else {
|
|
fail "Failed to delete history AOF"
|
|
}
|
|
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.5${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
}
|
|
|
|
test "AOF enable/disable auto gc" {
|
|
r config set aof-disable-auto-gc yes
|
|
|
|
r bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
|
|
r bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
|
|
# We can see four history AOFs (Evolved from two BASE and two INCR)
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.7.base.rdb seq 7 type b}
|
|
{file appendonly.aof.2.incr.aof seq 2 type h}
|
|
{file appendonly.aof.6.base.rdb seq 6 type h}
|
|
{file appendonly.aof.1.incr.aof seq 1 type h}
|
|
{file appendonly.aof.5.base.rdb seq 5 type h}
|
|
{file appendonly.aof.3.incr.aof seq 3 type i}
|
|
}
|
|
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.5${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.6${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
|
|
r config set aof-disable-auto-gc no
|
|
|
|
# Auto gc success
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.7.base.rdb seq 7 type b}
|
|
{file appendonly.aof.3.incr.aof seq 3 type i}
|
|
}
|
|
|
|
# wait bio delete history
|
|
wait_for_condition 1000 10 {
|
|
[check_file_exist $aof_dirpath "${aof_basename}.5${::base_aof_sufix}${::rdb_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.6${::base_aof_sufix}${::rdb_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.1${::incr_aof_sufix}${::aof_format_suffix}"] == 0 &&
|
|
[check_file_exist $aof_dirpath "${aof_basename}.2${::incr_aof_sufix}${::aof_format_suffix}"] == 0
|
|
} else {
|
|
fail "Failed to delete history AOF"
|
|
}
|
|
}
|
|
|
|
test "AOF can produce consecutive sequence number after reload" {
|
|
# Current manifest, BASE seq 7 and INCR seq 3
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.7.base.rdb seq 7 type b}
|
|
{file appendonly.aof.3.incr.aof seq 3 type i}
|
|
}
|
|
|
|
r debug loadaof
|
|
|
|
# Trigger AOFRW
|
|
r bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
|
|
# Now BASE seq is 8 and INCR seq is 4
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.8.base.rdb seq 8 type b}
|
|
{file appendonly.aof.4.incr.aof seq 4 type i}
|
|
}
|
|
}
|
|
|
|
test "AOF enable during BGSAVE will not write data util AOFRW finish" {
|
|
r config set appendonly no
|
|
r config set save ""
|
|
r config set rdb-key-save-delay 10000000
|
|
|
|
r set k1 v1
|
|
r bgsave
|
|
|
|
wait_for_condition 1000 10 {
|
|
[s rdb_bgsave_in_progress] eq 1
|
|
} else {
|
|
fail "bgsave did not start in time"
|
|
}
|
|
|
|
# Make server.aof_rewrite_scheduled = 1
|
|
r config set appendonly yes
|
|
assert_equal [s aof_rewrite_scheduled] 1
|
|
|
|
# Not open new INCR aof
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.8.base.rdb seq 8 type b}
|
|
{file appendonly.aof.4.incr.aof seq 4 type i}
|
|
}
|
|
|
|
r set k2 v2
|
|
r debug loadaof
|
|
|
|
# Both k1 and k2 lost
|
|
assert_equal 0 [r exists k1]
|
|
assert_equal 0 [r exists k2]
|
|
|
|
set total_forks [s total_forks]
|
|
assert_equal [s rdb_bgsave_in_progress] 1
|
|
r config set rdb-key-save-delay 0
|
|
catch {exec kill -9 [get_child_pid 0]}
|
|
wait_for_condition 1000 10 {
|
|
[s rdb_bgsave_in_progress] eq 0
|
|
} else {
|
|
fail "bgsave did not stop in time"
|
|
}
|
|
|
|
# Make sure AOFRW was scheduled
|
|
wait_for_condition 1000 10 {
|
|
[s total_forks] == [expr $total_forks + 1]
|
|
} else {
|
|
fail "aof rewrite did not scheduled"
|
|
}
|
|
waitForBgrewriteaof r
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.9.base.rdb seq 9 type b}
|
|
{file appendonly.aof.5.incr.aof seq 5 type i}
|
|
}
|
|
|
|
r set k3 v3
|
|
r debug loadaof
|
|
assert_equal v3 [r get k3]
|
|
}
|
|
|
|
test "AOF will trigger limit when AOFRW fails many times" {
|
|
# Clear all data and trigger a successful AOFRW, so we can let
|
|
# server.aof_current_size equal to 0
|
|
r flushall
|
|
r bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
|
|
r config set rdb-key-save-delay 10000000
|
|
# Let us trigger AOFRW easily
|
|
r config set auto-aof-rewrite-percentage 1
|
|
r config set auto-aof-rewrite-min-size 1kb
|
|
|
|
# Set a key so that AOFRW can be delayed
|
|
r set k v
|
|
|
|
# Let AOFRW fail 3 times, this will trigger AOFRW limit
|
|
r bgrewriteaof
|
|
catch {exec kill -9 [get_child_pid 0]}
|
|
waitForBgrewriteaof r
|
|
|
|
r bgrewriteaof
|
|
catch {exec kill -9 [get_child_pid 0]}
|
|
waitForBgrewriteaof r
|
|
|
|
r bgrewriteaof
|
|
catch {exec kill -9 [get_child_pid 0]}
|
|
waitForBgrewriteaof r
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.10.base.rdb seq 10 type b}
|
|
{file appendonly.aof.6.incr.aof seq 6 type i}
|
|
{file appendonly.aof.7.incr.aof seq 7 type i}
|
|
{file appendonly.aof.8.incr.aof seq 8 type i}
|
|
{file appendonly.aof.9.incr.aof seq 9 type i}
|
|
}
|
|
|
|
# Write 1KB data to trigger AOFRW
|
|
r set x [string repeat x 1024]
|
|
|
|
# Make sure we have limit log
|
|
wait_for_condition 1000 50 {
|
|
[count_log_message 0 "triggered the limit"] == 1
|
|
} else {
|
|
fail "aof rewrite did not trigger limit"
|
|
}
|
|
assert_equal [status r aof_rewrite_in_progress] 0
|
|
|
|
# No new INCR AOF be created
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.10.base.rdb seq 10 type b}
|
|
{file appendonly.aof.6.incr.aof seq 6 type i}
|
|
{file appendonly.aof.7.incr.aof seq 7 type i}
|
|
{file appendonly.aof.8.incr.aof seq 8 type i}
|
|
{file appendonly.aof.9.incr.aof seq 9 type i}
|
|
}
|
|
|
|
# Turn off auto rewrite
|
|
r config set auto-aof-rewrite-percentage 0
|
|
r config set rdb-key-save-delay 0
|
|
catch {exec kill -9 [get_child_pid 0]}
|
|
wait_for_condition 1000 10 {
|
|
[s aof_rewrite_in_progress] eq 0
|
|
} else {
|
|
fail "aof rewrite did not stop in time"
|
|
}
|
|
|
|
# We can still manually execute AOFRW immediately
|
|
r bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
|
|
# Can create New INCR AOF
|
|
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.10${::incr_aof_sufix}${::aof_format_suffix}"]
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.11.base.rdb seq 11 type b}
|
|
{file appendonly.aof.10.incr.aof seq 10 type i}
|
|
}
|
|
|
|
set d1 [r debug digest]
|
|
r debug loadaof
|
|
set d2 [r debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
|
|
start_server {overrides {aof-use-rdb-preamble {yes} appendonly {no} save {}}} {
|
|
set dir [get_redis_dir]
|
|
set aof_basename "appendonly.aof"
|
|
set aof_dirname "appendonlydir"
|
|
set aof_dirpath "$dir/$aof_dirname"
|
|
set aof_manifest_name "$aof_basename$::manifest_suffix"
|
|
set aof_manifest_file "$dir/$aof_dirname/$aof_manifest_name"
|
|
|
|
set master [srv 0 client]
|
|
set master_host [srv 0 host]
|
|
set master_port [srv 0 port]
|
|
|
|
test "AOF will open a temporary INCR AOF to accumulate data until the first AOFRW success when AOF is dynamically enabled" {
|
|
r config set save ""
|
|
# Increase AOFRW execution time to give us enough time to kill it
|
|
r config set rdb-key-save-delay 10000000
|
|
|
|
# Start write load
|
|
set load_handle0 [start_write_load $master_host $master_port 10]
|
|
|
|
wait_for_condition 50 100 {
|
|
[r dbsize] > 0
|
|
} else {
|
|
fail "No write load detected."
|
|
}
|
|
|
|
# Enable AOF will trigger an initialized AOFRW
|
|
r config set appendonly yes
|
|
# Let AOFRW fail
|
|
assert_equal 1 [s aof_rewrite_in_progress]
|
|
set pid1 [get_child_pid 0]
|
|
catch {exec kill -9 $pid1}
|
|
|
|
# Wait for AOFRW to exit and delete temp incr aof
|
|
wait_for_condition 1000 100 {
|
|
[count_log_message 0 "Removing the temp incr aof file"] == 1
|
|
} else {
|
|
fail "temp aof did not delete"
|
|
}
|
|
|
|
# Make sure manifest file is not created
|
|
assert_equal 0 [check_file_exist $aof_dirpath $aof_manifest_name]
|
|
# Make sure BASE AOF is not created
|
|
assert_equal 0 [check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::rdb_format_suffix}"]
|
|
|
|
# Make sure the next AOFRW has started
|
|
wait_for_condition 1000 50 {
|
|
[s aof_rewrite_in_progress] == 1
|
|
} else {
|
|
fail "aof rewrite did not scheduled"
|
|
}
|
|
|
|
# Do a successful AOFRW
|
|
set total_forks [s total_forks]
|
|
r config set rdb-key-save-delay 0
|
|
catch {exec kill -9 [get_child_pid 0]}
|
|
|
|
# Make sure the next AOFRW has started
|
|
wait_for_condition 1000 10 {
|
|
[s total_forks] == [expr $total_forks + 1]
|
|
} else {
|
|
fail "aof rewrite did not scheduled"
|
|
}
|
|
waitForBgrewriteaof r
|
|
|
|
assert_equal 2 [count_log_message 0 "Removing the temp incr aof file"]
|
|
|
|
# BASE and INCR AOF are successfully created
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.rdb seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
stop_write_load $load_handle0
|
|
wait_load_handlers_disconnected
|
|
|
|
set d1 [r debug digest]
|
|
r debug loadaof
|
|
set d2 [r debug digest]
|
|
assert {$d1 eq $d2}
|
|
|
|
# Dynamic disable AOF again
|
|
r config set appendonly no
|
|
|
|
# Disabling AOF does not delete previous AOF files
|
|
r debug loadaof
|
|
set d2 [r debug digest]
|
|
assert {$d1 eq $d2}
|
|
|
|
assert_equal 0 [s rdb_changes_since_last_save]
|
|
r config set rdb-key-save-delay 10000000
|
|
set load_handle0 [start_write_load $master_host $master_port 10]
|
|
wait_for_condition 50 100 {
|
|
[s rdb_changes_since_last_save] > 0
|
|
} else {
|
|
fail "No write load detected."
|
|
}
|
|
|
|
# Re-enable AOF
|
|
r config set appendonly yes
|
|
|
|
# Let AOFRW fail
|
|
assert_equal 1 [s aof_rewrite_in_progress]
|
|
set pid1 [get_child_pid 0]
|
|
catch {exec kill -9 $pid1}
|
|
|
|
# Wait for AOFRW to exit and delete temp incr aof
|
|
wait_for_condition 1000 100 {
|
|
[count_log_message 0 "Removing the temp incr aof file"] == 3
|
|
} else {
|
|
fail "temp aof did not delete 3 times"
|
|
}
|
|
|
|
# Make sure no new incr AOF was created
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.rdb seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
|
|
# Make sure the next AOFRW has started
|
|
wait_for_condition 1000 50 {
|
|
[s aof_rewrite_in_progress] == 1
|
|
} else {
|
|
fail "aof rewrite did not scheduled"
|
|
}
|
|
|
|
# Do a successful AOFRW
|
|
set total_forks [s total_forks]
|
|
r config set rdb-key-save-delay 0
|
|
catch {exec kill -9 [get_child_pid 0]}
|
|
|
|
wait_for_condition 1000 10 {
|
|
[s total_forks] == [expr $total_forks + 1]
|
|
} else {
|
|
fail "aof rewrite did not scheduled"
|
|
}
|
|
waitForBgrewriteaof r
|
|
|
|
assert_equal 4 [count_log_message 0 "Removing the temp incr aof file"]
|
|
|
|
# New BASE and INCR AOF are successfully created
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.2.base.rdb seq 2 type b}
|
|
{file appendonly.aof.2.incr.aof seq 2 type i}
|
|
}
|
|
|
|
stop_write_load $load_handle0
|
|
wait_load_handlers_disconnected
|
|
|
|
set d1 [r debug digest]
|
|
r debug loadaof
|
|
set d2 [r debug digest]
|
|
assert {$d1 eq $d2}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Test Part 3
|
|
#
|
|
# Test if INCR AOF offset information is as expected
|
|
test {Multi Part AOF writes start offset in the manifest} {
|
|
set aof_dirpath "$server_path/$aof_dirname"
|
|
set aof_manifest_file "$server_path/$aof_dirname/${aof_basename}$::manifest_suffix"
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
# The manifest file has startoffset now
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.rdb seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i startoffset 0}
|
|
}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF won't add the offset of incr AOF from old version} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
|
}
|
|
|
|
start_server_aof [list dir $server_path] {
|
|
assert_equal 1 [is_alive [srv pid]]
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_equal v1 [$client get k1]
|
|
assert_equal v2 [$client get k2]
|
|
|
|
$client set k3 v3
|
|
catch {$client shutdown}
|
|
|
|
# Should not add offset to the manifest since we also don't know the right
|
|
# starting replication of them.
|
|
set fp [open $aof_manifest_file r]
|
|
set content [read $fp]
|
|
close $fp
|
|
assert ![regexp {startoffset} $content]
|
|
|
|
# The manifest file still have information from the old version
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.aof seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i}
|
|
}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can update master_repl_offset with only startoffset info} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i startoffset 100\n"
|
|
}
|
|
|
|
start_server [list overrides [list dir $server_path appendonly yes ]] {
|
|
wait_done_loading r
|
|
r select 0
|
|
assert_equal v1 [r get k1]
|
|
assert_equal v2 [r get k2]
|
|
|
|
# After loading AOF, redis will update the replication offset based on
|
|
# the information of the last INCR AOF, to avoid the rollback of the
|
|
# start offset of new INCR AOF. If the INCR file doesn't have an end offset
|
|
# info, redis will calculate the replication offset by the start offset
|
|
# plus the file size.
|
|
set file_size [file size $aof_incr1_file]
|
|
set offset [expr $file_size + 100]
|
|
assert_equal $offset [s master_repl_offset]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF can update master_repl_offset with endoffset info} {
|
|
create_aof $aof_dirpath $aof_base1_file {
|
|
append_to_aof [formatCommand set k1 v1]
|
|
}
|
|
|
|
create_aof $aof_dirpath $aof_incr1_file {
|
|
append_to_aof [formatCommand set k2 v2]
|
|
}
|
|
|
|
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
|
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
|
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i startoffset 100 endoffset 200\n"
|
|
}
|
|
|
|
start_server [list overrides [list dir $server_path appendonly yes ]] {
|
|
wait_done_loading r
|
|
r select 0
|
|
assert_equal v1 [r get k1]
|
|
assert_equal v2 [r get k2]
|
|
|
|
# If the INCR file has an end offset, redis directly uses it as replication offset
|
|
assert_equal 200 [s master_repl_offset]
|
|
|
|
# We should reset endoffset in manifest file
|
|
set fp [open $aof_manifest_file r]
|
|
set content [read $fp]
|
|
close $fp
|
|
assert ![regexp {endoffset} $content]
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {Multi Part AOF will add the end offset if we close gracefully the AOF} {
|
|
start_server_aof [list dir $server_path] {
|
|
set client [redis [srv host] [srv port] 0 $::tls]
|
|
wait_done_loading $client
|
|
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.rdb seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i startoffset 0}
|
|
}
|
|
|
|
$client set k1 v1
|
|
$client set k2 v2
|
|
# Close AOF gracefully when stopping appendonly, we should add endoffset
|
|
# in the manifest file, 'endoffset' should be 2 since writing 2 commands
|
|
r config set appendonly no
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.1.base.rdb seq 1 type b}
|
|
{file appendonly.aof.1.incr.aof seq 1 type i startoffset 0 endoffset 2}
|
|
}
|
|
r config set appendonly yes
|
|
waitForBgrewriteaof $client
|
|
|
|
$client set k3 v3
|
|
# Close AOF gracefully when shutting down server, we should add endoffset
|
|
# in the manifest file, 'endoffset' should be 3 since writing 3 commands
|
|
catch {$client shutdown}
|
|
assert_aof_manifest_content $aof_manifest_file {
|
|
{file appendonly.aof.2.base.rdb seq 2 type b}
|
|
{file appendonly.aof.2.incr.aof seq 2 type i startoffset 2 endoffset 3}
|
|
}
|
|
}
|
|
|
|
clean_aof_persistence $aof_dirpath
|
|
}
|
|
|
|
test {INCR AOF has accurate start offset when AOFRW} {
|
|
start_server [list overrides [list dir $server_path appendonly yes ]] {
|
|
r config set auto-aof-rewrite-percentage 0
|
|
|
|
# Start write load to let the master_repl_offset continue increasing
|
|
# since appendonly is enabled
|
|
set load_handle0 [start_write_load [srv 0 host] [srv 0 port] 10]
|
|
wait_for_condition 50 100 {
|
|
[r dbsize] > 0
|
|
} else {
|
|
fail "No write load detected."
|
|
}
|
|
|
|
# We obtain the master_repl_offset at the time of bgrewriteaof by pausing
|
|
# the redis process, sending pipeline commands, and then resuming the process
|
|
set rd [redis_deferring_client]
|
|
pause_process [srv 0 pid]
|
|
set buf "info replication\r\n"
|
|
append buf "bgrewriteaof\r\n"
|
|
$rd write $buf
|
|
$rd flush
|
|
resume_process [srv 0 pid]
|
|
# Read the replication offset and the start of the bgrewriteaof
|
|
regexp {master_repl_offset:(\d+)} [$rd read] -> offset1
|
|
assert_match {*rewriting started*} [$rd read]
|
|
$rd close
|
|
|
|
# Get the start offset from the manifest file after bgrewriteaof
|
|
waitForBgrewriteaof r
|
|
set fp [open $aof_manifest_file r]
|
|
set content [read $fp]
|
|
close $fp
|
|
set offset2 [lindex [regexp -inline {startoffset (\d+)} $content] 1]
|
|
|
|
# The start offset of INCR AOF should be the same as master_repl_offset
|
|
# when we trigger bgrewriteaof
|
|
assert {$offset1 == $offset2}
|
|
stop_write_load $load_handle0
|
|
wait_load_handlers_disconnected
|
|
}
|
|
}
|
|
}
|