[CRASH] Redis 7.0.12 crashed in zsetDel

[SarthakSahu]

Crashed in https://github.com/redis/redis/blob/7.0.12/src/t_zset.c#L1473

=== REDIS BUG REPORT START: Cut & paste starting from here ===
9:S 29 Apr 2024 04:19:48.479 # === ASSERTION FAILED ===
9:S 29 Apr 2024 04:19:48.479 # ==> t_zset.c:1473 'retval' is not true

------ STACK TRACE ------

Backtrace:
/usr/bin/redis-server *:6379 [cluster][0x48cee7]
/usr/bin/redis-server *:6379 [cluster](zsetDel+0x2d)[0x48f3bd]
/usr/bin/redis-server *:6379 [cluster](zremCommand+0x82)[0x48ff42]
/usr/bin/redis-server *:6379 [cluster](call+0xc0)[0x44cce0]
/usr/bin/redis-server *:6379 [cluster](execCommand+0x1b1)[0x4a52f1]
/usr/bin/redis-server *:6379 [cluster](call+0xc0)[0x44cce0]
/usr/bin/redis-server *:6379 [cluster](processCommand+0x875)[0x44f2c5]
/usr/bin/redis-server *:6379 [cluster](processCommandAndResetClient+0x1c)[0x4650ec]
/usr/bin/redis-server *:6379 [cluster](processInputBuffer+0xd0)[0x467b50]
/usr/bin/redis-server *:6379 [cluster](readQueryFromClient+0x2a8)[0x46ae98]
/usr/bin/redis-server *:6379 [cluster][0x506bd8]
/usr/bin/redis-server *:6379 [cluster](aeProcessEvents+0x1c9)[0x444619]
/usr/bin/redis-server *:6379 [cluster](aeMain+0x1d)[0x4449ad]
/usr/bin/redis-server *:6379 [cluster](main+0x30f)[0x4408df]
/lib64/libc.so.6(__libc_start_main+0xe5)[0x7fbe9f3d7d85]
/usr/bin/redis-server *:6379 [cluster](_start+0x2e)[0x440f2e]

------ INFO OUTPUT ------
# Server
redis_version:7.0.12
redis_git_sha1:e9be2f9a
redis_git_dirty:0
redis_build_id:7a0155b8bb55f6e9
redis_mode:cluster
os:Linux 4.18.0-513.24.1.el8_9.x86_64 x86_64
arch_bits:64
monotonic_clock:POSIX clock_gettime
multiplexing_api:epoll
atomicvar_api:c11-builtin
gcc_version:8.5.0
process_id:9
process_supervised:no
run_id:3ab29ebac6d2a5ec56e8b4e0f62406ef4020c3a3
tcp_port:6379
server_time_usec:1714335588479728
uptime_in_seconds:40241
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:3059556
executable:/usr/bin/redis-server
config_file:/redisdb/conf/server.conf
io_threads_active:0

# Clients
connected_clients:3
cluster_connections:26
maxclients:10000
client_recent_max_input_buffer:20480
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0

# Memory
used_memory:2859252072
used_memory_human:2.66G
used_memory_rss:3042582528
used_memory_rss_human:2.83G
used_memory_peak:2859290920
used_memory_peak_human:2.66G
used_memory_peak_perc:100.00%
used_memory_overhead:923467928
used_memory_startup:1669168
used_memory_dataset:1935784144
used_memory_dataset_perc:67.74%
allocator_allocated:2859502664
allocator_active:2979434496
allocator_resident:3021402112
total_system_memory:67246452736
total_system_memory_human:62.63G
used_memory_lua:31744
used_memory_vm_eval:31744
used_memory_lua_human:31.00K
used_memory_scripts_eval:0
number_of_cached_scripts:0
number_of_functions:0
number_of_libraries:0
used_memory_vm_functions:32768
used_memory_vm_total:64512
used_memory_vm_total_human:63.00K
used_memory_functions:184
used_memory_scripts:184
used_memory_scripts_human:184B
maxmemory:28991029248
maxmemory_human:27.00G
maxmemory_policy:noeviction
allocator_frag_ratio:1.04
allocator_frag_bytes:119931832
allocator_rss_ratio:1.01
allocator_rss_bytes:41967616
rss_overhead_ratio:1.01
rss_overhead_bytes:21180416
mem_fragmentation_ratio:1.06
mem_fragmentation_bytes:183339256
mem_not_counted_for_evict:0
mem_replication_backlog:594623400
mem_total_replication_buffers:594616000
mem_clients_slaves:0
mem_clients_normal:35592
mem_cluster_links:28288
mem_aof_buffer:0
mem_allocator:jemalloc-5.2.1
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

# Persistence
loading:0
async_loading:0
current_cow_peak:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:4350950
rdb_bgsave_in_progress:0
rdb_last_save_time:1714295347
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_saves:0
rdb_last_cow_size:0
rdb_last_load_keys_expired:0
rdb_last_load_keys_loaded:4642889
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_rewrites:0
aof_rewrites_consecutive_failures:0
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0

# Stats
total_connections_received:3306
total_commands_processed:3263886
instantaneous_ops_per_sec:92
total_net_input_bytes:1489975963
total_net_output_bytes:975590575
total_net_repl_input_bytes:1485974642
total_net_repl_output_bytes:0
instantaneous_input_kbps:17.52
instantaneous_output_kbps:14.09
instantaneous_input_repl_kbps:17.46
instantaneous_output_repl_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
evicted_clients:0
total_eviction_exceeded_time:0
current_eviction_exceeded_time:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
pubsubshard_channels:0
latest_fork_usec:0
total_forks:0
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
total_active_defrag_time:0
current_active_defrag_time:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:13
dump_payload_sanitizations:0
total_reads_processed:1361058
total_writes_processed:271056
io_threaded_reads_processed:0
io_threaded_writes_processed:0
reply_buffer_shrinks:14993
reply_buffer_expands:29434

# Replication
role:slave
master_host:10.234.64.229
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_read_repl_offset:2922241695
slave_repl_offset:2922241457
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:09cb64333ec3a480cd2c1525838b55eddca2439b
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:2922241457
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1610612736
repl_backlog_first_byte_offset:2329496266
repl_backlog_histlen:592745192

# CPU
used_cpu_sys:109.258162
used_cpu_user:146.800325
used_cpu_sys_children:0.341688
used_cpu_user_children:1.036207
used_cpu_sys_main_thread:107.410617
used_cpu_user_main_thread:145.344967

# Modules
module:name=auditlog,ver=3,api=1,filters=0,usedby=[],using=[],options=[handle-io-errors|handle-repl-async-load]
module:name=NtasBcsSdb,ver=1,api=1,filters=0,usedby=[],using=[],options=[]

# Commandstats
cmdstat_multi:calls=431386,usec=222788,usec_per_call=0.52,rejected_calls=0,failed_calls=0
cmdstat_acl|log:calls=399120,usec=1058821,usec_per_call=2.65,rejected_calls=0,failed_calls=0
cmdstat_del:calls=215691,usec=2998097,usec_per_call=13.90,rejected_calls=0,failed_calls=0
cmdstat_cluster|info:calls=40088,usec=1990710,usec_per_call=49.66,rejected_calls=11,failed_calls=0
cmdstat_auth:calls=3306,usec=18958,usec_per_call=5.73,rejected_calls=0,failed_calls=0
cmdstat_hdel:calls=214891,usec=2397296,usec_per_call=11.16,rejected_calls=0,failed_calls=0
cmdstat_info:calls=179719,usec=10900191,usec_per_call=60.65,rejected_calls=0,failed_calls=0
cmdstat_select:calls=1,usec=1,usec_per_call=1.00,rejected_calls=0,failed_calls=0
cmdstat_client|setname:calls=3,usec=18,usec_per_call=6.00,rejected_calls=0,failed_calls=0
cmdstat_zadd:calls=215695,usec=6580856,usec_per_call=30.51,rejected_calls=0,failed_calls=0
cmdstat_zrem:calls=215690,usec=2038551,usec_per_call=9.45,rejected_calls=0,failed_calls=0
cmdstat_hset:calls=904875,usec=23631561,usec_per_call=26.12,rejected_calls=0,failed_calls=0
cmdstat_exec:calls=431385,usec=24885358,usec_per_call=57.69,rejected_calls=0,failed_calls=0
cmdstat_ping:calls=12036,usec=15149,usec_per_call=1.26,rejected_calls=2,failed_calls=0

# Errorstats
errorstat_LOADING:count=13

# Latencystats
latency_percentiles_usec_multi:p50=0.001,p99=2.007,p99.9=10.047
latency_percentiles_usec_acl|log:p50=2.007,p99=7.007,p99.9=30.079
latency_percentiles_usec_del:p50=13.055,p99=39.167,p99.9=72.191
latency_percentiles_usec_cluster|info:p50=48.127,p99=88.063,p99.9=144.383
latency_percentiles_usec_auth:p50=3.007,p99=22.015,p99.9=61.183
latency_percentiles_usec_hdel:p50=10.047,p99=30.079,p99.9=53.247
latency_percentiles_usec_info:p50=47.103,p99=156.671,p99.9=207.871
latency_percentiles_usec_select:p50=1.003,p99=1.003,p99.9=1.003
latency_percentiles_usec_client|setname:p50=3.007,p99=12.031,p99.9=12.031
latency_percentiles_usec_zadd:p50=25.087,p99=103.423,p99.9=174.079
latency_percentiles_usec_zrem:p50=8.031,p99=34.047,p99.9=83.455
latency_percentiles_usec_hset:p50=19.071,p99=105.471,p99.9=169.983
latency_percentiles_usec_exec:p50=46.079,p99=179.199,p99.9=274.431
latency_percentiles_usec_ping:p50=1.003,p99=3.007,p99.9=20.095

# Cluster
cluster_enabled:1

# Keyspace
db0:keys=4642900,expires=0,avg_ttl=0

------ CLIENT LIST OUTPUT ------
id=15 addr=[::1]:53692 laddr=[::1]:6379 fd=38 name=local_monitor age=40212 idle=3 flags=N db=0 sub=0 psub=0 ssub=0 multi=-1 qbuf=0 qbuf-free=0 argv-mem=0 multi-mem=0 rbs=2048 rbp=1024 obl=0 oll=0 omem=0 tot-mem=2824 events=r cmd=info user=probe-user redir=-1 resp=2
id=24 addr=10.234.64.229:6379 laddr=10.254.96.3:53376 fd=33 name= age=40206 idle=0 flags=Mx db=0 sub=0 psub=0 ssub=0 multi=2 qbuf=238 qbuf-free=40716 argv-mem=4 multi-mem=209 rbs=1024 rbp=44 obl=0 oll=0 omem=0 tot-mem=43037 events=r cmd=exec user=(superuser) redir=-1 resp=2
id=25 addr=[::1]:55222 laddr=[::1]:6379 fd=39 name= age=40206 idle=0 flags=N db=0 sub=0 psub=0 ssub=0 multi=-1 qbuf=0 qbuf-free=20474 argv-mem=0 multi-mem=0 rbs=8192 rbp=5604 obl=0 oll=0 omem=0 tot-mem=29440 events=r cmd=info user=probe-user redir=-1 resp=2

------ CURRENT CLIENT INFO ------
id=24 addr=10.234.64.229:6379 laddr=10.254.96.3:53376 fd=33 name= age=40206 idle=0 flags=Mx db=0 sub=0 psub=0 ssub=0 multi=2 qbuf=238 qbuf-free=40716 argv-mem=4 multi-mem=209 rbs=1024 rbp=44 obl=0 oll=0 omem=0 tot-mem=43037 events=r cmd=exec user=(superuser) redir=-1 resp=2
argv[0]: '\"ZREM\"'
argv[1]: '\"sdb:bcs0:partyList:spi:{2Lm}\"'
argv[2]: '\"5000014555\"'
9:S 29 Apr 2024 04:19:48.482 # key 'sdb:bcs0:partyList:spi:{2Lm}' found in DB containing the following object:
9:S 29 Apr 2024 04:19:48.482 # Object type: 3
9:S 29 Apr 2024 04:19:48.482 # Object encoding: 7
9:S 29 Apr 2024 04:19:48.482 # Object refcount: 1

------ MODULES INFO OUTPUT ------

------ CONFIG DEBUG OUTPUT ------
repl-diskless-load disabled
io-threads-do-reads no
activedefrag no
replica-read-only yes
lazyfree-lazy-eviction no
client-query-buffer-limit 1gb
lazyfree-lazy-server-del no
proto-max-bulk-len 512mb
repl-diskless-sync yes
lazyfree-lazy-expire no
list-compress-depth 0
io-threads 1
sanitize-dump-payload no
lazyfree-lazy-user-del no
lazyfree-lazy-user-flush no
slave-read-only yes

------ FAST MEMORY TEST ------
9:S 29 Apr 2024 04:19:48.482 # Bio thread for job type #0 terminated
9:S 29 Apr 2024 04:19:48.482 # Bio thread for job type #1 terminated
9:S 29 Apr 2024 04:19:48.483 # Bio thread for job type #2 terminated
*** Preparing to test memory region 88a000 (2306048 bytes)
*** Preparing to test memory region c05000 (2072576 bytes)
*** Preparing to test memory region 7fbdb87ff000 (1006632960 bytes)
*** Preparing to test memory region 7fbdf4800000 (8388608 bytes)
*** Preparing to test memory region 7fbdf5000000 (2533494784 bytes)
*** Preparing to test memory region 7fbe90200000 (8388608 bytes)
*** Preparing to test memory region 7fbe90a80000 (47710208 bytes)
*** Preparing to test memory region 7fbe93a00000 (8388608 bytes)
*** Preparing to test memory region 7fbe94200000 (4194304 bytes)
*** Preparing to test memory region 7fbe9476a000 (8388608 bytes)
*** Preparing to test memory region 7fbe94f6b000 (8388608 bytes)
*** Preparing to test memory region 7fbe9576c000 (8388608 bytes)
*** Preparing to test memory region 7fbe95f6d000 (8388608 bytes)
*** Preparing to test memory region 7fbe9676e000 (8388608 bytes)
*** Preparing to test memory region 7fbe9721c000 (8192 bytes)
*** Preparing to test memory region 7fbe97c25000 (4096 bytes)
*** Preparing to test memory region 7fbe99c91000 (4096 bytes)
*** Preparing to test memory region 7fbe9a159000 (4096 bytes)
*** Preparing to test memory region 7fbe9ab45000 (4096 bytes)
*** Preparing to test memory region 7fbe9b25f000 (8192 bytes)
*** Preparing to test memory region 7fbe9b80b000 (12288 bytes)
*** Preparing to test memory region 7fbe9c56a000 (274432 bytes)
*** Preparing to test memory region 7fbe9c9d6000 (524288 bytes)
*** Preparing to test memory region 7fbe9cc75000 (262144 bytes)
*** Preparing to test memory region 7fbe9d2df000 (561152 bytes)
*** Preparing to test memory region 7fbe9e2bf000 (200704 bytes)
*** Preparing to test memory region 7fbe9e500000 (11534336 bytes)
*** Preparing to test memory region 7fbe9f005000 (1572864 bytes)
*** Preparing to test memory region 7fbe9f39c000 (4096 bytes)
*** Preparing to test memory region 7fbe9f75e000 (16384 bytes)
*** Preparing to test memory region 7fbe9f97e000 (16384 bytes)
*** Preparing to test memory region 7fbe9fe67000 (20480 bytes)
*** Preparing to test memory region 7fbea00ff000 (4096 bytes)
*** Preparing to test memory region 7fbea08d7000 (1572864 bytes)
*** Preparing to test memory region 7fbea0aaa000 (32768 bytes)
*** Preparing to test memory region 7fbea0ab4000 (8192 bytes)
.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O
Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.

=== REDIS BUG REPORT END. Make sure to include from START to END. ===