diff --git a/deps/xredis-gtid b/deps/xredis-gtid index 46b26f83348..3f23791340a 160000 --- a/deps/xredis-gtid +++ b/deps/xredis-gtid @@ -1 +1 @@ -Subproject commit 46b26f83348962168836d6ca1713e9051a2a6d2a +Subproject commit 3f23791340ad49aa70f324d22b12232a42672ef5 diff --git a/src/Makefile b/src/Makefile index ff01f494f37..7ed46ae7d8a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -454,9 +454,11 @@ distclean: clean .PHONY: distclean test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) + @$(MAKE) -C ../deps/xredis-gtid build-test-modules @(cd ..; ./runtest) test-asan: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) + @$(MAKE) -C ../deps/xredis-gtid build-test-modules @(cd ..; ./runtest --tags -nosanitizer --asan) test-modules: $(REDIS_SERVER_NAME) diff --git a/src/ctrip_swap_rocks.c b/src/ctrip_swap_rocks.c index f29bfae318c..5f4c943dd3d 100644 --- a/src/ctrip_swap_rocks.c +++ b/src/ctrip_swap_rocks.c @@ -1152,20 +1152,29 @@ sds intervalInfo(sds info, char* rocksdb_stats) { static uint64_t rocksUsedDbSize(rocks *rocks) { char *err = NULL; - uint64_t used_db_size = 0, total_used_db_size = 0; + uint64_t total_used_db_size = 0; const char *begin_key = "\x0", *end_key = "\xff"; const size_t begin_key_len = 1, end_key_len = 1; for (int i = 0; i < CF_COUNT; i++) { + uint64_t sst_size = 0; rocksdb_column_family_handle_t *handle = rocks->cf_handles[i]; if (handle == NULL) continue; rocksdb_approximate_sizes_cf(rocks->db,handle,1,&begin_key,&begin_key_len, - &end_key,&end_key_len,&used_db_size,&err); + &end_key,&end_key_len,&sst_size,&err); if (err != NULL) { serverLog(LL_WARNING, "rocksdb_approximate_sizes_cf failed: %s",err); continue; } - total_used_db_size += used_db_size; + total_used_db_size += sst_size; + + /* Also include blob file size for this CF, which is not + * accounted for by rocksdb_approximate_sizes_cf. */ + uint64_t blob_size = 0; + if (!rocksdb_property_int_cf(rocks->db, handle, + "rocksdb.total-blob-file-size", &blob_size)) { + total_used_db_size += blob_size; + } } return total_used_db_size; diff --git a/src/server.c b/src/server.c index 4145236ebf7..479671184d5 100644 --- a/src/server.c +++ b/src/server.c @@ -3516,6 +3516,7 @@ void initServer(void) { memset(server.master_uuid,'0',CONFIG_RUN_ID_SIZE); server.master_uuid[CONFIG_RUN_ID_SIZE] = 0; server.master_uuid_len = CONFIG_RUN_ID_SIZE; + serverGtidEmbeddedClear(); server.gtid_executed = gtidSetNew(); gtidSetCurrentUuidSetUpdate(server.gtid_executed,server.uuid,server.uuid_len); server.gtid_lost = gtidSetNew(); @@ -4205,7 +4206,7 @@ void call(client *c, int flags) { /* Call propagate() only if at least one of AOF / replication * propagation is needed. Note that modules commands handle replication * in an explicit way, so we never replicate them automatically. */ - if (propagate_flags != PROPAGATE_NONE && !(c->cmd->flags & CMD_MODULE)) + if (propagate_flags != PROPAGATE_NONE && !(c->cmd->flags & CMD_MODULE) && c->cmd->proc != gtidCommand) propagate(c->cmd,c->db->id,c->argv,c->argc,propagate_flags); } @@ -4229,9 +4230,13 @@ void call(client *c, int flags) { * in case the nested MULTI/EXEC. * * And if the array contains only one command, no need to - * wrap it, since the single command is atomic. */ + * wrap it, since the single command is atomic. + * + * gtidCommand only rewrites what the inner command already + * decided to propagate; do not add another MULTI/EXEC layer. */ if (server.also_propagate.numops > 1 && !(c->cmd->flags & CMD_MODULE) && + c->cmd->proc != gtidCommand && !(c->flags & CLIENT_MULTI) && !(flags & CMD_CALL_NOWRAP)) { @@ -4254,6 +4259,7 @@ void call(client *c, int flags) { } } redisOpArrayFree(&server.also_propagate); + serverGtidEmbeddedClear(); } server.also_propagate = prev_also_propagate; diff --git a/src/server.h b/src/server.h index 374f6b66f87..f8b4fad4e45 100644 --- a/src/server.h +++ b/src/server.h @@ -1744,6 +1744,13 @@ struct redisServer { long long gtid_sync_stat[GTID_SYNC_TYPES]; int gtid_gaplog_enabled; gtidGaplog* gtid_gap_log; + /* Caller-supplied GTID identity for the current GTID-wrapped command. + * Set by gtidCommand(), so that rewriting ops + * with the caller's uuid/gno instead of auto-allocating a new one. */ + char *gtid_embedded_uuid; + size_t gtid_embedded_uuid_len; + gno_t gtid_embedded_gno; /* 0 means "not set" (gno starts from 1) */ + int gtid_embedded_dbid; /* importing mode */ mstime_t importing_end_time; /* in milliseconds */ diff --git a/tests/helpers/gen_write_load.tcl b/tests/helpers/gen_write_load.tcl index 59b91e4265c..87833bdba16 100644 --- a/tests/helpers/gen_write_load.tcl +++ b/tests/helpers/gen_write_load.tcl @@ -2,17 +2,44 @@ source tests/support/redis.tcl set ::tlsdir "tests/tls" -proc gen_write_load {host port seconds tls db} { +# Continuously sends SET commands to the server. If key is omitted, a random key +# is used for every SET command. The value is always random. +proc gen_write_load {host port seconds tls db {key ""} {size 0} {sleep 0}} { set start_time [clock seconds] set r [redis $host $port 1 $tls] $r client setname LOAD_HANDLER $r select $db + + # fixed size value + if {$size != 0} { + set value [string repeat "x" $size] + } + while 1 { - $r set [expr rand()] [expr rand()] + if {$size == 0} { + set value [expr rand()] + } + + if {$key == ""} { + if {[catch {$r set [expr rand()] $value} err]} { + exit 0 + } + } else { + if {[catch {$r set $key $value} err]} { + exit 0 + } + } if {[clock seconds]-$start_time > $seconds} { exit 0 } + if {$sleep ne 0} { + after $sleep + } } } -gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] +if {[llength $argv] >= 8} { + gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] [lindex $argv 5] [lindex $argv 6] [lindex $argv 7] +} else { + gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] +} diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 55408844116..a9803d41796 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -548,10 +548,11 @@ proc find_valgrind_errors {stderr on_termination} { } # Execute a background process writing random data for the specified number -# of seconds to the specified Redis instance. -proc start_write_load {host port seconds} { +# of seconds to the specified Redis instance. If key is omitted, a random key +# is used for every SET command. +proc start_write_load {host port seconds {key ""} {size 0} {sleep 0}} { set tclsh [info nameofexecutable] - exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds $::tls $::target_db & + exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds $::tls $::target_db $key $size $sleep & } # Stop a process generating write load executed with start_write_load. diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index af0d85ba16a..31e8bf7d73e 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -13,8 +13,12 @@ source tests/support/util.tcl source tests/support/gtid.tcl set ::gtid_tests { - gtid/gtid + gtid/6_x/gtid + gtid/6_x/aof + gtid/6_x/sync + gtid/gtid gtid/gtid_seq + gtid/gtid_replicate gtid/replication-psync gtid/sync gtid/xsync