From 52d179035f07b9caa01a517fa96d69fd9df0c0ce Mon Sep 17 00:00:00 2001 From: Warrick <1016weicheng@gmail.com> Date: Fri, 3 Jul 2026 16:53:15 +0800 Subject: [PATCH 1/3] [fix] united gtid repo of 6.x and 8.x --- deps/xredis-gtid | 2 +- src/Makefile | 8 ++++++-- src/replication.c | 6 +++--- src/server.c | 10 ++++++++-- src/server.h | 7 +++++++ tests/helpers/gen_write_load.tcl | 33 +++++++++++++++++++++++++++++--- tests/modules/propagate.c | 29 ++++++++++++++++++++++++++++ tests/support/util.tcl | 7 ++++--- tests/test_helper.tcl | 6 +++++- 9 files changed, 93 insertions(+), 15 deletions(-) diff --git a/deps/xredis-gtid b/deps/xredis-gtid index 46b26f83348..47f1665c758 160000 --- a/deps/xredis-gtid +++ b/deps/xredis-gtid @@ -1 +1 @@ -Subproject commit 46b26f83348962168836d6ca1713e9051a2a6d2a +Subproject commit 47f1665c75870d50a132155fe4ac33fa51e5d90e diff --git a/src/Makefile b/src/Makefile index ff01f494f37..9c511dbc38a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -452,11 +452,15 @@ distclean: clean -(rm -f .make-*) .PHONY: distclean +.PHONY: build-test-modules -test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) +build-test-modules: + @$(MAKE) -C ../tests/modules + +test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) build-test-modules @(cd ..; ./runtest) -test-asan: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) +test-asan: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) build-test-modules @(cd ..; ./runtest --tags -nosanitizer --asan) test-modules: $(REDIS_SERVER_NAME) diff --git a/src/replication.c b/src/replication.c index ed8e6fdfc50..e28d6fd2506 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2754,9 +2754,9 @@ void replicationSetMaster(char *ip, int port) { /* Update oom_score_adj */ setOOMScoreAdj(-1); - /* Force our slaves to resync with us as well. They may hopefully be able - * to partially resync with us, but we can notify the replid change. */ - disconnectSlaves(); + /* Disconnecting here prematurely causes a cascade reconnect storm + * before we even know the sync outcome, + * which breaks topology changes under load. See Redis commit cee3d67f. */ cancelReplicationHandshake(0); /* Before destroying our master state, create a cached master using * our own parameters, to later PSYNC with the new master. */ diff --git a/src/server.c b/src/server.c index 4145236ebf7..479671184d5 100644 --- a/src/server.c +++ b/src/server.c @@ -3516,6 +3516,7 @@ void initServer(void) { memset(server.master_uuid,'0',CONFIG_RUN_ID_SIZE); server.master_uuid[CONFIG_RUN_ID_SIZE] = 0; server.master_uuid_len = CONFIG_RUN_ID_SIZE; + serverGtidEmbeddedClear(); server.gtid_executed = gtidSetNew(); gtidSetCurrentUuidSetUpdate(server.gtid_executed,server.uuid,server.uuid_len); server.gtid_lost = gtidSetNew(); @@ -4205,7 +4206,7 @@ void call(client *c, int flags) { /* Call propagate() only if at least one of AOF / replication * propagation is needed. Note that modules commands handle replication * in an explicit way, so we never replicate them automatically. */ - if (propagate_flags != PROPAGATE_NONE && !(c->cmd->flags & CMD_MODULE)) + if (propagate_flags != PROPAGATE_NONE && !(c->cmd->flags & CMD_MODULE) && c->cmd->proc != gtidCommand) propagate(c->cmd,c->db->id,c->argv,c->argc,propagate_flags); } @@ -4229,9 +4230,13 @@ void call(client *c, int flags) { * in case the nested MULTI/EXEC. * * And if the array contains only one command, no need to - * wrap it, since the single command is atomic. */ + * wrap it, since the single command is atomic. + * + * gtidCommand only rewrites what the inner command already + * decided to propagate; do not add another MULTI/EXEC layer. */ if (server.also_propagate.numops > 1 && !(c->cmd->flags & CMD_MODULE) && + c->cmd->proc != gtidCommand && !(c->flags & CLIENT_MULTI) && !(flags & CMD_CALL_NOWRAP)) { @@ -4254,6 +4259,7 @@ void call(client *c, int flags) { } } redisOpArrayFree(&server.also_propagate); + serverGtidEmbeddedClear(); } server.also_propagate = prev_also_propagate; diff --git a/src/server.h b/src/server.h index 374f6b66f87..f8b4fad4e45 100644 --- a/src/server.h +++ b/src/server.h @@ -1744,6 +1744,13 @@ struct redisServer { long long gtid_sync_stat[GTID_SYNC_TYPES]; int gtid_gaplog_enabled; gtidGaplog* gtid_gap_log; + /* Caller-supplied GTID identity for the current GTID-wrapped command. + * Set by gtidCommand(), so that rewriting ops + * with the caller's uuid/gno instead of auto-allocating a new one. */ + char *gtid_embedded_uuid; + size_t gtid_embedded_uuid_len; + gno_t gtid_embedded_gno; /* 0 means "not set" (gno starts from 1) */ + int gtid_embedded_dbid; /* importing mode */ mstime_t importing_end_time; /* in milliseconds */ diff --git a/tests/helpers/gen_write_load.tcl b/tests/helpers/gen_write_load.tcl index 59b91e4265c..87833bdba16 100644 --- a/tests/helpers/gen_write_load.tcl +++ b/tests/helpers/gen_write_load.tcl @@ -2,17 +2,44 @@ source tests/support/redis.tcl set ::tlsdir "tests/tls" -proc gen_write_load {host port seconds tls db} { +# Continuously sends SET commands to the server. If key is omitted, a random key +# is used for every SET command. The value is always random. +proc gen_write_load {host port seconds tls db {key ""} {size 0} {sleep 0}} { set start_time [clock seconds] set r [redis $host $port 1 $tls] $r client setname LOAD_HANDLER $r select $db + + # fixed size value + if {$size != 0} { + set value [string repeat "x" $size] + } + while 1 { - $r set [expr rand()] [expr rand()] + if {$size == 0} { + set value [expr rand()] + } + + if {$key == ""} { + if {[catch {$r set [expr rand()] $value} err]} { + exit 0 + } + } else { + if {[catch {$r set $key $value} err]} { + exit 0 + } + } if {[clock seconds]-$start_time > $seconds} { exit 0 } + if {$sleep ne 0} { + after $sleep + } } } -gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] +if {[llength $argv] >= 8} { + gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] [lindex $argv 5] [lindex $argv 6] [lindex $argv 7] +} else { + gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] +} diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index ac04d4f9d2d..ff315a40555 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -149,6 +149,25 @@ int propagateTestSimpleCommand(RedisModuleCtx *ctx, RedisModuleString **argv, in return REDISMODULE_OK; } +int propagateTestSingleCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModule_Replicate(ctx,"INCR","c","single-counter"); + RedisModule_ReplyWithSimpleString(ctx,"OK"); + return REDISMODULE_OK; +} + +int propagateTestNoReplicateCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModule_ReplyWithSimpleString(ctx,"OK"); + return REDISMODULE_OK; +} + int propagateTestMixedCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); @@ -237,6 +256,16 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) "",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"propagate-test.single", + propagateTestSingleCommand, + "",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"propagate-test.noreplicate", + propagateTestNoReplicateCommand, + "",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"propagate-test.mixed", propagateTestMixedCommand, "",1,1,1) == REDISMODULE_ERR) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 55408844116..a9803d41796 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -548,10 +548,11 @@ proc find_valgrind_errors {stderr on_termination} { } # Execute a background process writing random data for the specified number -# of seconds to the specified Redis instance. -proc start_write_load {host port seconds} { +# of seconds to the specified Redis instance. If key is omitted, a random key +# is used for every SET command. +proc start_write_load {host port seconds {key ""} {size 0} {sleep 0}} { set tclsh [info nameofexecutable] - exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds $::tls $::target_db & + exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds $::tls $::target_db $key $size $sleep & } # Stop a process generating write load executed with start_write_load. diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index af0d85ba16a..7def6fe2070 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -13,8 +13,12 @@ source tests/support/util.tcl source tests/support/gtid.tcl set ::gtid_tests { - gtid/gtid + gtid/6_x/gtid + gtid/6_x/aof + gtid/6_x/sync + gtid/gtid gtid/gtid_seq + gtid/gtid_module_replicate gtid/replication-psync gtid/sync gtid/xsync From 2786e5443ba9ecf42b14e5587eeb12ada881dee1 Mon Sep 17 00:00:00 2001 From: Warrick <1016weicheng@gmail.com> Date: Tue, 14 Jul 2026 14:50:42 +0800 Subject: [PATCH 2/3] [test] review fix --- src/Makefile | 10 ++++------ src/replication.c | 6 +++--- tests/modules/propagate.c | 29 ----------------------------- tests/test_helper.tcl | 2 +- 4 files changed, 8 insertions(+), 39 deletions(-) diff --git a/src/Makefile b/src/Makefile index 9c511dbc38a..7ed46ae7d8a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -452,15 +452,13 @@ distclean: clean -(rm -f .make-*) .PHONY: distclean -.PHONY: build-test-modules -build-test-modules: - @$(MAKE) -C ../tests/modules - -test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) build-test-modules +test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) + @$(MAKE) -C ../deps/xredis-gtid build-test-modules @(cd ..; ./runtest) -test-asan: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) build-test-modules +test-asan: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) + @$(MAKE) -C ../deps/xredis-gtid build-test-modules @(cd ..; ./runtest --tags -nosanitizer --asan) test-modules: $(REDIS_SERVER_NAME) diff --git a/src/replication.c b/src/replication.c index e28d6fd2506..ed8e6fdfc50 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2754,9 +2754,9 @@ void replicationSetMaster(char *ip, int port) { /* Update oom_score_adj */ setOOMScoreAdj(-1); - /* Disconnecting here prematurely causes a cascade reconnect storm - * before we even know the sync outcome, - * which breaks topology changes under load. See Redis commit cee3d67f. */ + /* Force our slaves to resync with us as well. They may hopefully be able + * to partially resync with us, but we can notify the replid change. */ + disconnectSlaves(); cancelReplicationHandshake(0); /* Before destroying our master state, create a cached master using * our own parameters, to later PSYNC with the new master. */ diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index ff315a40555..ac04d4f9d2d 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -149,25 +149,6 @@ int propagateTestSimpleCommand(RedisModuleCtx *ctx, RedisModuleString **argv, in return REDISMODULE_OK; } -int propagateTestSingleCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) -{ - REDISMODULE_NOT_USED(argv); - REDISMODULE_NOT_USED(argc); - - RedisModule_Replicate(ctx,"INCR","c","single-counter"); - RedisModule_ReplyWithSimpleString(ctx,"OK"); - return REDISMODULE_OK; -} - -int propagateTestNoReplicateCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) -{ - REDISMODULE_NOT_USED(argv); - REDISMODULE_NOT_USED(argc); - - RedisModule_ReplyWithSimpleString(ctx,"OK"); - return REDISMODULE_OK; -} - int propagateTestMixedCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); @@ -256,16 +237,6 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) "",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; - if (RedisModule_CreateCommand(ctx,"propagate-test.single", - propagateTestSingleCommand, - "",1,1,1) == REDISMODULE_ERR) - return REDISMODULE_ERR; - - if (RedisModule_CreateCommand(ctx,"propagate-test.noreplicate", - propagateTestNoReplicateCommand, - "",1,1,1) == REDISMODULE_ERR) - return REDISMODULE_ERR; - if (RedisModule_CreateCommand(ctx,"propagate-test.mixed", propagateTestMixedCommand, "",1,1,1) == REDISMODULE_ERR) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 7def6fe2070..31e8bf7d73e 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -18,7 +18,7 @@ set ::gtid_tests { gtid/6_x/sync gtid/gtid gtid/gtid_seq - gtid/gtid_module_replicate + gtid/gtid_replicate gtid/replication-psync gtid/sync gtid/xsync From 425720777152bcc297067acde87a800252e680ea Mon Sep 17 00:00:00 2001 From: Warrick <1016weicheng@gmail.com> Date: Fri, 21 Aug 2026 12:40:31 +0800 Subject: [PATCH 3/3] [fix] add size of blobfiles into metric of swap_used_db_size --- deps/xredis-gtid | 2 +- src/ctrip_swap_rocks.c | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/deps/xredis-gtid b/deps/xredis-gtid index 47f1665c758..3f23791340a 160000 --- a/deps/xredis-gtid +++ b/deps/xredis-gtid @@ -1 +1 @@ -Subproject commit 47f1665c75870d50a132155fe4ac33fa51e5d90e +Subproject commit 3f23791340ad49aa70f324d22b12232a42672ef5 diff --git a/src/ctrip_swap_rocks.c b/src/ctrip_swap_rocks.c index f29bfae318c..5f4c943dd3d 100644 --- a/src/ctrip_swap_rocks.c +++ b/src/ctrip_swap_rocks.c @@ -1152,20 +1152,29 @@ sds intervalInfo(sds info, char* rocksdb_stats) { static uint64_t rocksUsedDbSize(rocks *rocks) { char *err = NULL; - uint64_t used_db_size = 0, total_used_db_size = 0; + uint64_t total_used_db_size = 0; const char *begin_key = "\x0", *end_key = "\xff"; const size_t begin_key_len = 1, end_key_len = 1; for (int i = 0; i < CF_COUNT; i++) { + uint64_t sst_size = 0; rocksdb_column_family_handle_t *handle = rocks->cf_handles[i]; if (handle == NULL) continue; rocksdb_approximate_sizes_cf(rocks->db,handle,1,&begin_key,&begin_key_len, - &end_key,&end_key_len,&used_db_size,&err); + &end_key,&end_key_len,&sst_size,&err); if (err != NULL) { serverLog(LL_WARNING, "rocksdb_approximate_sizes_cf failed: %s",err); continue; } - total_used_db_size += used_db_size; + total_used_db_size += sst_size; + + /* Also include blob file size for this CF, which is not + * accounted for by rocksdb_approximate_sizes_cf. */ + uint64_t blob_size = 0; + if (!rocksdb_property_int_cf(rocks->db, handle, + "rocksdb.total-blob-file-size", &blob_size)) { + total_used_db_size += blob_size; + } } return total_used_db_size;