[v6.1] possible deadlock in timer_delete_sync

[syzbot]

Hello,

syzbot found the following issue on:

HEAD commit: 7dc732d24ff7 Linux 6.1.123
git tree: linux-6.1.y
console output: https://syzkaller.appspot.com/x/log.txt?x=115ca4b0580000
kernel config: https://syzkaller.appspot.com/x/.config?x=da1827eaa51b65c3
dashboard link: https://syzkaller.appspot.com/bug?extid=62d18996f32177e2a4db
compiler: Debian clang version 15.0.6, GNU ld (GNU Binutils for Debian) 2.40
userspace arch: arm64

Unfortunately, I don't have any reproducer for this issue yet.

Downloadable assets:
disk image: https://storage.googleapis.com/syzbot-assets/b77a36eba7b4/disk-7dc732d2.raw.xz
vmlinux: https://storage.googleapis.com/syzbot-assets/f5334562da28/vmlinux-7dc732d2.xz
kernel image: https://storage.googleapis.com/syzbot-assets/f0a16f9a500c/Image-7dc732d2.gz.xz

IMPORTANT: if you fix the issue, please add the following tag to the commit:
Reported-by: syzbot+62d189...@syzkaller.appspotmail.com

======================================================
WARNING: possible circular locking dependency detected
6.1.123-syzkaller #0 Not tainted
------------------------------------------------------
syz.0.296/5811 is trying to acquire lock:
ffff0000c30370a8 ((&sq->pending_timer)){+.-.}-{0:0}, at: timer_delete_sync+0x74/0x210 kernel/time/timer.c:1448

but task is already holding lock:
ffff80001ce537b0 (&blkcg->lock){....}-{2:2}, at: spin_lock include/linux/spinlock.h:351 [inline]
ffff80001ce537b0 (&blkcg->lock){....}-{2:2}, at: blkcg_deactivate_policy+0x1ac/0x4a8 block/blk-cgroup.c:1493

which lock already depends on the new lock.


the existing dependency chain (in reverse order) is:

-> #2 (&blkcg->lock){....}-{2:2}:
__raw_spin_lock include/linux/spinlock_api_smp.h:133 [inline]
_raw_spin_lock+0x54/0x6c kernel/locking/spinlock.c:154
spin_lock include/linux/spinlock.h:351 [inline]
blkg_create+0x9f4/0x1158 block/blk-cgroup.c:320
blkcg_init_disk+0xd0/0x318 block/blk-cgroup.c:1259
__alloc_disk_node+0x26c/0x484 block/genhd.c:1412
__blk_alloc_disk+0x40/0xbc block/genhd.c:1451
brd_alloc+0x324/0x610 drivers/block/brd.c:424
brd_init+0x134/0x1a8 drivers/block/brd.c:529
do_one_initcall+0x260/0xacc init/main.c:1298
do_initcall_level+0x154/0x214 init/main.c:1371
do_initcalls+0x58/0xac init/main.c:1387
do_basic_setup+0x8c/0xa0 init/main.c:1406
kernel_init_freeable+0x3a4/0x528 init/main.c:1626
kernel_init+0x24/0x29c init/main.c:1514
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:864

-> #1 (&q->queue_lock){..-.}-{2:2}:
__raw_spin_lock_irq include/linux/spinlock_api_smp.h:119 [inline]
_raw_spin_lock_irq+0x70/0x9c kernel/locking/spinlock.c:170
spin_lock_irq include/linux/spinlock.h:376 [inline]
throtl_pending_timer_fn+0x104/0xdcc block/blk-throttle.c:1200
call_timer_fn+0x1c0/0xa1c kernel/time/timer.c:1504
expire_timers kernel/time/timer.c:1549 [inline]
__run_timers+0x554/0x718 kernel/time/timer.c:1820
run_timer_softirq+0x7c/0x114 kernel/time/timer.c:1833
handle_softirqs+0x318/0xd58 kernel/softirq.c:571
__do_softirq+0x14/0x20 kernel/softirq.c:605
____do_softirq+0x14/0x20 arch/arm64/kernel/irq.c:80
call_on_irq_stack+0x24/0x4c arch/arm64/kernel/entry.S:893
do_softirq_own_stack+0x20/0x2c arch/arm64/kernel/irq.c:85
invoke_softirq kernel/softirq.c:452 [inline]
__irq_exit_rcu+0x264/0x4d4 kernel/softirq.c:654
irq_exit_rcu+0x14/0x84 kernel/softirq.c:666
__el1_irq arch/arm64/kernel/entry-common.c:472 [inline]
el1_interrupt+0x38/0x68 arch/arm64/kernel/entry-common.c:486
el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:491
el1h_64_irq+0x64/0x68 arch/arm64/kernel/entry.S:581
do_raw_spin_lock+0x0/0x358
spin_lock include/linux/spinlock.h:351 [inline]
wp_page_copy+0x554/0x1970 mm/memory.c:3189
do_wp_page+0xb10/0xf44
handle_pte_fault mm/memory.c:5047 [inline]
__handle_mm_fault mm/memory.c:5171 [inline]
handle_mm_fault+0x19a4/0x3d38 mm/memory.c:5292
__do_page_fault arch/arm64/mm/fault.c:499 [inline]
do_page_fault+0x330/0x890 arch/arm64/mm/fault.c:583
do_mem_abort+0x74/0x200 arch/arm64/mm/fault.c:803
el0_da+0x70/0x184 arch/arm64/kernel/entry-common.c:515
el0t_64_sync_handler+0xcc/0xf0 arch/arm64/kernel/entry-common.c:658
el0t_64_sync+0x18c/0x190 arch/arm64/kernel/entry.S:585

-> #0 ((&sq->pending_timer)){+.-.}-{0:0}:
check_prev_add kernel/locking/lockdep.c:3090 [inline]
check_prevs_add kernel/locking/lockdep.c:3209 [inline]
validate_chain kernel/locking/lockdep.c:3825 [inline]
__lock_acquire+0x3338/0x7680 kernel/locking/lockdep.c:5049
lock_acquire+0x26c/0x7cc kernel/locking/lockdep.c:5662
timer_delete_sync+0x9c/0x210 kernel/time/timer.c:1448
del_timer_sync include/linux/timer.h:198 [inline]
throtl_pd_free+0x20/0x48 block/blk-throttle.c:493
blkcg_deactivate_policy+0x2cc/0x4a8 block/blk-cgroup.c:1497
blk_throtl_exit+0x9c/0x13c block/blk-throttle.c:2415
blkcg_init_disk+0x2a4/0x318 block/blk-cgroup.c:1283
__alloc_disk_node+0x26c/0x484 block/genhd.c:1412
__blk_mq_alloc_disk+0x114/0x1e4 block/blk-mq.c:4146
nbd_dev_add+0x314/0x938 drivers/block/nbd.c:1812
nbd_genl_connect+0x8fc/0x17f4 drivers/block/nbd.c:1995
genl_family_rcv_msg_doit net/netlink/genetlink.c:756 [inline]
genl_family_rcv_msg net/netlink/genetlink.c:833 [inline]
genl_rcv_msg+0x948/0xc2c net/netlink/genetlink.c:850
netlink_rcv_skb+0x20c/0x3b8 net/netlink/af_netlink.c:2493
genl_rcv+0x38/0x50 net/netlink/genetlink.c:861
netlink_unicast_kernel net/netlink/af_netlink.c:1311 [inline]
netlink_unicast+0x65c/0x898 net/netlink/af_netlink.c:1337
netlink_sendmsg+0x834/0xb18 net/netlink/af_netlink.c:1859
sock_sendmsg_nosec net/socket.c:718 [inline]
__sock_sendmsg net/socket.c:730 [inline]
____sys_sendmsg+0x55c/0x848 net/socket.c:2519
___sys_sendmsg net/socket.c:2573 [inline]
__sys_sendmsg+0x26c/0x33c net/socket.c:2602
__do_sys_sendmsg net/socket.c:2611 [inline]
__se_sys_sendmsg net/socket.c:2609 [inline]
__arm64_sys_sendmsg+0x80/0x94 net/socket.c:2609
__invoke_syscall arch/arm64/kernel/syscall.c:38 [inline]
invoke_syscall+0x98/0x2bc arch/arm64/kernel/syscall.c:52
el0_svc_common+0x138/0x258 arch/arm64/kernel/syscall.c:140
do_el0_svc+0x58/0x13c arch/arm64/kernel/syscall.c:204
el0_svc+0x58/0x168 arch/arm64/kernel/entry-common.c:637
el0t_64_sync_handler+0x84/0xf0 arch/arm64/kernel/entry-common.c:655
el0t_64_sync+0x18c/0x190 arch/arm64/kernel/entry.S:585

other info that might help us debug this:

Chain exists of:
(&sq->pending_timer) --> &q->queue_lock --> &blkcg->lock

Possible unsafe locking scenario:

CPU0 CPU1
---- ----
lock(&blkcg->lock);
lock(&q->queue_lock);
lock(&blkcg->lock);
lock((&sq->pending_timer));

*** DEADLOCK ***

4 locks held by syz.0.296/5811:
#0: ffff80001811f250 (cb_lock){++++}-{3:3}, at: genl_rcv+0x28/0x50 net/netlink/genetlink.c:860
#1: ffff80001811f108 (genl_mutex){+.+.}-{3:3}, at: genl_lock net/netlink/genetlink.c:33 [inline]
#1: ffff80001811f108 (genl_mutex){+.+.}-{3:3}, at: genl_rcv_msg+0x114/0xc2c net/netlink/genetlink.c:848
#2: ffff0000cf2b6778 (&q->queue_lock){..-.}-{2:2}, at: spin_lock_irq include/linux/spinlock.h:376 [inline]
#2: ffff0000cf2b6778 (&q->queue_lock){..-.}-{2:2}, at: blkcg_deactivate_policy+0xf4/0x4a8 block/blk-cgroup.c:1486
#3: ffff80001ce537b0 (&blkcg->lock){....}-{2:2}, at: spin_lock include/linux/spinlock.h:351 [inline]
#3: ffff80001ce537b0 (&blkcg->lock){....}-{2:2}, at: blkcg_deactivate_policy+0x1ac/0x4a8 block/blk-cgroup.c:1493

stack backtrace:
CPU: 1 PID: 5811 Comm: syz.0.296 Not tainted 6.1.123-syzkaller #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 09/13/2024
Call trace:
dump_backtrace+0x1c8/0x1f4 arch/arm64/kernel/stacktrace.c:158
show_stack+0x2c/0x3c arch/arm64/kernel/stacktrace.c:165
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0x108/0x170 lib/dump_stack.c:106
dump_stack+0x1c/0x58 lib/dump_stack.c:113
print_circular_bug+0x150/0x1b8 kernel/locking/lockdep.c:2048
check_noncircular+0x2cc/0x378 kernel/locking/lockdep.c:2170
check_prev_add kernel/locking/lockdep.c:3090 [inline]
check_prevs_add kernel/locking/lockdep.c:3209 [inline]
validate_chain kernel/locking/lockdep.c:3825 [inline]
__lock_acquire+0x3338/0x7680 kernel/locking/lockdep.c:5049
lock_acquire+0x26c/0x7cc kernel/locking/lockdep.c:5662
timer_delete_sync+0x9c/0x210 kernel/time/timer.c:1448
del_timer_sync include/linux/timer.h:198 [inline]
throtl_pd_free+0x20/0x48 block/blk-throttle.c:493
blkcg_deactivate_policy+0x2cc/0x4a8 block/blk-cgroup.c:1497
blk_throtl_exit+0x9c/0x13c block/blk-throttle.c:2415
blkcg_init_disk+0x2a4/0x318 block/blk-cgroup.c:1283
__alloc_disk_node+0x26c/0x484 block/genhd.c:1412
__blk_mq_alloc_disk+0x114/0x1e4 block/blk-mq.c:4146
nbd_dev_add+0x314/0x938 drivers/block/nbd.c:1812
nbd_genl_connect+0x8fc/0x17f4 drivers/block/nbd.c:1995
genl_family_rcv_msg_doit net/netlink/genetlink.c:756 [inline]
genl_family_rcv_msg net/netlink/genetlink.c:833 [inline]
genl_rcv_msg+0x948/0xc2c net/netlink/genetlink.c:850
netlink_rcv_skb+0x20c/0x3b8 net/netlink/af_netlink.c:2493
genl_rcv+0x38/0x50 net/netlink/genetlink.c:861
netlink_unicast_kernel net/netlink/af_netlink.c:1311 [inline]
netlink_unicast+0x65c/0x898 net/netlink/af_netlink.c:1337
netlink_sendmsg+0x834/0xb18 net/netlink/af_netlink.c:1859
sock_sendmsg_nosec net/socket.c:718 [inline]
__sock_sendmsg net/socket.c:730 [inline]
____sys_sendmsg+0x55c/0x848 net/socket.c:2519
___sys_sendmsg net/socket.c:2573 [inline]
__sys_sendmsg+0x26c/0x33c net/socket.c:2602
__do_sys_sendmsg net/socket.c:2611 [inline]
__se_sys_sendmsg net/socket.c:2609 [inline]
__arm64_sys_sendmsg+0x80/0x94 net/socket.c:2609
__invoke_syscall arch/arm64/kernel/syscall.c:38 [inline]
invoke_syscall+0x98/0x2bc arch/arm64/kernel/syscall.c:52
el0_svc_common+0x138/0x258 arch/arm64/kernel/syscall.c:140
do_el0_svc+0x58/0x13c arch/arm64/kernel/syscall.c:204
el0_svc+0x58/0x168 arch/arm64/kernel/entry-common.c:637
el0t_64_sync_handler+0x84/0xf0 arch/arm64/kernel/entry-common.c:655
el0t_64_sync+0x18c/0x190 arch/arm64/kernel/entry.S:585
nbd: failed to add new device


---
This report is generated by a bot. It may contain errors.
See https://goo.gl/tpsmEJ for more information about syzbot.
syzbot engineers can be reached at syzk...@googlegroups.com.

syzbot will keep track of this issue. See:
https://goo.gl/tpsmEJ#status for how to communicate with syzbot.

If the report is already addressed, let syzbot know by replying with:
#syz fix: exact-commit-title

If you want to overwrite report's subsystems, reply with:
#syz set subsystems: new-subsystem
(See the list of subsystem names on the web dashboard)

If the report is a duplicate of another one, reply with:
#syz dup: exact-subject-of-another-report

If you want to undo deduplication, reply with:
#syz undup

[syzbot]

syzbot has found a reproducer for the following issue on:

HEAD commit: 7dc732d24ff7 Linux 6.1.123
git tree: linux-6.1.y

console output: https://syzkaller.appspot.com/x/log.txt?x=1433f9c4580000

kernel config: https://syzkaller.appspot.com/x/.config?x=da1827eaa51b65c3
dashboard link: https://syzkaller.appspot.com/bug?extid=62d18996f32177e2a4db
compiler: Debian clang version 15.0.6, GNU ld (GNU Binutils for Debian) 2.40
userspace arch: arm64

syz repro: https://syzkaller.appspot.com/x/repro.syz?x=169aa4b0580000
C reproducer: https://syzkaller.appspot.com/x/repro.c?x=13237418580000

Downloadable assets:
disk image: https://storage.googleapis.com/syzbot-assets/b77a36eba7b4/disk-7dc732d2.raw.xz
vmlinux: https://storage.googleapis.com/syzbot-assets/f5334562da28/vmlinux-7dc732d2.xz
kernel image: https://storage.googleapis.com/syzbot-assets/f0a16f9a500c/Image-7dc732d2.gz.xz

IMPORTANT: if you fix the issue, please add the following tag to the commit:
Reported-by: syzbot+62d189...@syzkaller.appspotmail.com

do_el0_svc+0x58/0x13c arch/arm64/kernel/syscall.c:204
el0_svc+0x58/0x168 arch/arm64/kernel/entry-common.c:637
el0t_64_sync_handler+0x84/0xf0 arch/arm64/kernel/entry-common.c:655
el0t_64_sync+0x18c/0x190 arch/arm64/kernel/entry.S:585

======================================================
WARNING: possible circular locking dependency detected
6.1.123-syzkaller #0 Not tainted
------------------------------------------------------

syz-executor425/4313 is trying to acquire lock:
ffff0000c4be10a8 ((&sq->pending_timer)){+.-.}-{0:0}, at: timer_delete_sync+0x74/0x210 kernel/time/timer.c:1448

arch_local_irq_restore arch/arm64/include/asm/irqflags.h:122 [inline]
call_rcu+0x624/0xa40 kernel/rcu/tree.c:2880
destroy_inode fs/inode.c:316 [inline]
evict+0x7ac/0x894 fs/inode.c:720
iput_final fs/inode.c:1834 [inline]
iput+0x7c0/0x8a4 fs/inode.c:1860
dentry_unlink_inode+0x37c/0x4bc fs/dcache.c:405
__dentry_kill+0x324/0x5e4 fs/dcache.c:611
dentry_kill+0xc8/0x250
dput+0x218/0x454 fs/dcache.c:918
find_next_child fs/libfs.c:265 [inline]
simple_recursive_removal+0x21c/0x6e0 fs/libfs.c:280
debugfs_remove+0x60/0x88 fs/debugfs/inode.c:778
blk_unregister_queue+0x160/0x19c block/blk-sysfs.c:926
del_gendisk+0x3a4/0x838 block/genhd.c:654
nbd_dev_remove drivers/block/nbd.c:264 [inline]
nbd_dev_remove_work+0x48/0xe0 drivers/block/nbd.c:280
process_one_work+0x7ac/0x1404 kernel/workqueue.c:2292
worker_thread+0x8e4/0xfec kernel/workqueue.c:2439
kthread+0x250/0x2d8 kernel/kthread.c:376
ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:864

4 locks held by syz-executor425/4313:

#0: ffff80001811f250 (cb_lock){++++}-{3:3}, at: genl_rcv+0x28/0x50 net/netlink/genetlink.c:860
#1: ffff80001811f108 (genl_mutex){+.+.}-{3:3}, at: genl_lock net/netlink/genetlink.c:33 [inline]
#1: ffff80001811f108 (genl_mutex){+.+.}-{3:3}, at: genl_rcv_msg+0x114/0xc2c net/netlink/genetlink.c:848

#2: ffff0000cee6a308 (&q->queue_lock){..-.}-{2:2}, at: spin_lock_irq include/linux/spinlock.h:376 [inline]
#2: ffff0000cee6a308 (&q->queue_lock){..-.}-{2:2}, at: blkcg_deactivate_policy+0xf4/0x4a8 block/blk-cgroup.c:1486

#3: ffff80001ce537b0 (&blkcg->lock){....}-{2:2}, at: spin_lock include/linux/spinlock.h:351 [inline]
#3: ffff80001ce537b0 (&blkcg->lock){....}-{2:2}, at: blkcg_deactivate_policy+0x1ac/0x4a8 block/blk-cgroup.c:1493

stack backtrace:

CPU: 1 PID: 4313 Comm: syz-executor425 Not tainted 6.1.123-syzkaller #0

If you want syzbot to run the reproducer, reply with:
#syz test: git://repo/address.git branch-or-commit-hash
If you attach or paste a git patch, syzbot will apply it before testing.