[Linux kernel bug] INFO: task hung in blk_mq_get_tag

From: Sam Sun
Date: Sun May 12 2024 - 22:39:13 EST

Next message: Chengming Zhou: "[PATCH v2 0/2] mm/ksm: fix some accounting problems"
Previous message: Nicolin Chen: "Re: [PATCH RFCv1 03/14] iommufd: Prepare for viommu structures and functions"
Next in thread: Hillf Danton: "Re: [Linux kernel bug] INFO: task hung in blk_mq_get_tag"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Dear developers and maintainers,

We encountered a task hung in function blk_mq_get_tag. It was tested
against the latest upstream kernel which was compiled by clang 14.
Kernel config and C repro are attached to this email. The bug report
is listed below.
```
Call Trace:
<TASK>
context_switch kernel/sched/core.c:5409 [inline]
__schedule+0xf19/0x15e0 kernel/sched/core.c:6746
__schedule_loop kernel/sched/core.c:6823 [inline]
schedule+0x147/0x310 kernel/sched/core.c:6838
io_schedule+0x87/0x100 kernel/sched/core.c:9044
blk_mq_get_tag+0x509/0xba0 block/blk-mq-tag.c:187
__blk_mq_alloc_requests+0xbc1/0x1710 block/blk-mq.c:499
blk_mq_alloc_request+0x513/0xbc0 block/blk-mq.c:599
scsi_alloc_request drivers/scsi/scsi_lib.c:1229 [inline]
scsi_execute_cmd+0x17a/0x1140 drivers/scsi/scsi_lib.c:304
scsi_vpd_inquiry drivers/scsi/scsi.c:312 [inline]
scsi_get_vpd_size+0x2e3/0x6b0 drivers/scsi/scsi.c:363
scsi_get_vpd_buf+0x89/0x460 drivers/scsi/scsi.c:433
scsi_attach_vpd+0xdc/0x5e0 drivers/scsi/scsi.c:501
scsi_rescan_device+0xd8/0x290 drivers/scsi/scsi_scan.c:1698
ata_scsi_dev_rescan+0x1fe/0x3c0 drivers/ata/libata-scsi.c:4764
process_one_work kernel/workqueue.c:3254 [inline]
process_scheduled_works+0x9c9/0x14a0 kernel/workqueue.c:3335
worker_thread+0x85c/0xd50 kernel/workqueue.c:3416
kthread+0x2ed/0x390 kernel/kthread.c:388
ret_from_fork+0x4b/0x80 arch/x86/kernel/process.c:147
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:244
</TASK>
INFO: task kworker/u10:2:53 blocked for more than 143 seconds.
Not tainted 6.9.0-rc6 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
task:kworker/u10:2 state:D stack:21688 pid:53 tgid:53 ppid:2
flags:0x00004000
Workqueue: writeback wb_workfn (flush-8:0)
Call Trace:
<TASK>
context_switch kernel/sched/core.c:5409 [inline]
__schedule+0xf19/0x15e0 kernel/sched/core.c:6746
__schedule_loop kernel/sched/core.c:6823 [inline]
schedule+0x147/0x310 kernel/sched/core.c:6838
io_schedule+0x87/0x100 kernel/sched/core.c:9044
bit_wait_io+0x12/0xc0 kernel/sched/wait_bit.c:209
__wait_on_bit+0xb0/0x320 kernel/sched/wait_bit.c:49
out_of_line_wait_on_bit+0x1c7/0x240 kernel/sched/wait_bit.c:64
wait_on_bit_io include/linux/wait_bit.h:101 [inline]
do_get_write_access+0x10c/0xcd0 fs/jbd2/transaction.c:1111
jbd2_journal_get_write_access+0x206/0x250 fs/jbd2/transaction.c:1260
__ext4_journal_get_write_access+0x1e6/0x6a0 fs/ext4/ext4_jbd2.c:239
ext4_mb_mark_context+0x30f/0xf50 fs/ext4/mballoc.c:4005
ext4_mb_mark_diskspace_used+0x39a/0x8e0 fs/ext4/mballoc.c:4122
ext4_mb_new_blocks+0x132b/0x4210 fs/ext4/mballoc.c:6223
ext4_ext_map_blocks+0x1da8/0x6b00 fs/ext4/extents.c:4317
ext4_map_blocks+0xa8c/0x1cb0 fs/ext4/inode.c:623
mpage_map_one_extent fs/ext4/inode.c:2163 [inline]
mpage_map_and_submit_extent fs/ext4/inode.c:2216 [inline]
ext4_do_writepages+0x16cb/0x3920 fs/ext4/inode.c:2679
ext4_writepages+0x210/0x3c0 fs/ext4/inode.c:2768
do_writepages+0x371/0x830 mm/page-writeback.c:2612
__writeback_single_inode+0xe2/0x660 fs/fs-writeback.c:1650
writeback_sb_inodes+0x831/0x1020 fs/fs-writeback.c:1941
__writeback_inodes_wb+0x11b/0x260 fs/fs-writeback.c:2012
wb_writeback+0x3c4/0x720 fs/fs-writeback.c:2119
wb_check_old_data_flush fs/fs-writeback.c:2223 [inline]
wb_do_writeback fs/fs-writeback.c:2276 [inline]
wb_workfn+0xa38/0xf30 fs/fs-writeback.c:2304
process_one_work kernel/workqueue.c:3254 [inline]
process_scheduled_works+0x9c9/0x14a0 kernel/workqueue.c:3335
worker_thread+0x85c/0xd50 kernel/workqueue.c:3416
kthread+0x2ed/0x390 kernel/kthread.c:388
ret_from_fork+0x4b/0x80 arch/x86/kernel/process.c:147
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:244
</TASK>
INFO: task kworker/u10:5:2795 blocked for more than 143 seconds.
Not tainted 6.9.0-rc6 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
task:kworker/u10:5 state:D stack:21200 pid:2795 tgid:2795 ppid:2
flags:0x00004000
Workqueue: writeback wb_workfn (flush-8:0)
Call Trace:
<TASK>
context_switch kernel/sched/core.c:5409 [inline]
__schedule+0xf19/0x15e0 kernel/sched/core.c:6746
__schedule_loop kernel/sched/core.c:6823 [inline]
schedule+0x147/0x310 kernel/sched/core.c:6838
io_schedule+0x87/0x100 kernel/sched/core.c:9044
blk_mq_get_tag+0x509/0xba0 block/blk-mq-tag.c:187
__blk_mq_alloc_requests+0xbc1/0x1710 block/blk-mq.c:499
blk_mq_get_new_requests block/blk-mq.c:2886 [inline]
blk_mq_submit_bio+0xcd2/0x1f00 block/blk-mq.c:2984
__submit_bio+0x23d/0x2e0 block/blk-core.c:621
__submit_bio_noacct_mq block/blk-core.c:700 [inline]
submit_bio_noacct_nocheck+0x9e1/0xc40 block/blk-core.c:729
ext4_io_submit+0xd8/0x140 fs/ext4/page-io.c:378
ext4_do_writepages+0x12f2/0x3920 fs/ext4/inode.c:2699
ext4_writepages+0x210/0x3c0 fs/ext4/inode.c:2768
do_writepages+0x371/0x830 mm/page-writeback.c:2612
__writeback_single_inode+0xe2/0x660 fs/fs-writeback.c:1650
writeback_sb_inodes+0x831/0x1020 fs/fs-writeback.c:1941
__writeback_inodes_wb+0x11b/0x260 fs/fs-writeback.c:2012
wb_writeback+0x3c4/0x720 fs/fs-writeback.c:2119
wb_check_background_flush fs/fs-writeback.c:2189 [inline]
wb_do_writeback fs/fs-writeback.c:2277 [inline]
wb_workfn+0xaef/0xf30 fs/fs-writeback.c:2304
process_one_work kernel/workqueue.c:3254 [inline]
process_scheduled_works+0x9c9/0x14a0 kernel/workqueue.c:3335
worker_thread+0x85c/0xd50 kernel/workqueue.c:3416
kthread+0x2ed/0x390 kernel/kthread.c:388
ret_from_fork+0x4b/0x80 arch/x86/kernel/process.c:147
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:244
</TASK>
INFO: task jbd2/sda-8:4502 blocked for more than 143 seconds.
Not tainted 6.9.0-rc6 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
task:jbd2/sda-8 state:D stack:24400 pid:4502 tgid:4502 ppid:2
flags:0x00004000
Call Trace:
<TASK>
context_switch kernel/sched/core.c:5409 [inline]
__schedule+0xf19/0x15e0 kernel/sched/core.c:6746
__schedule_loop kernel/sched/core.c:6823 [inline]
schedule+0x147/0x310 kernel/sched/core.c:6838
io_schedule+0x87/0x100 kernel/sched/core.c:9044
blk_mq_get_tag+0x509/0xba0 block/blk-mq-tag.c:187
__blk_mq_alloc_requests+0xbc1/0x1710 block/blk-mq.c:499
blk_mq_get_new_requests block/blk-mq.c:2886 [inline]
blk_mq_submit_bio+0xcd2/0x1f00 block/blk-mq.c:2984
__submit_bio+0x23d/0x2e0 block/blk-core.c:621
__submit_bio_noacct_mq block/blk-core.c:700 [inline]
submit_bio_noacct_nocheck+0x9e1/0xc40 block/blk-core.c:729
jbd2_journal_commit_transaction+0x2412/0x5500 fs/jbd2/commit.c:731
kjournald2+0x45f/0x870 fs/jbd2/journal.c:201
kthread+0x2ed/0x390 kernel/kthread.c:388
ret_from_fork+0x4b/0x80 arch/x86/kernel/process.c:147
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:244
</TASK>
INFO: task systemd-journal:4529 blocked for more than 143 seconds.
Not tainted 6.9.0-rc6 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
task:systemd-journal state:D stack:18872 pid:4529 tgid:4529 ppid:1
flags:0x00000004
Call Trace:
<TASK>
context_switch kernel/sched/core.c:5409 [inline]
__schedule+0xf19/0x15e0 kernel/sched/core.c:6746
__schedule_loop kernel/sched/core.c:6823 [inline]
schedule+0x147/0x310 kernel/sched/core.c:6838
io_schedule+0x87/0x100 kernel/sched/core.c:9044
bit_wait_io+0x12/0xc0 kernel/sched/wait_bit.c:209
__wait_on_bit+0xb0/0x320 kernel/sched/wait_bit.c:49
out_of_line_wait_on_bit+0x1c7/0x240 kernel/sched/wait_bit.c:64
wait_on_bit_io include/linux/wait_bit.h:101 [inline]
do_get_write_access+0x10c/0xcd0 fs/jbd2/transaction.c:1111
jbd2_journal_get_write_access+0x206/0x250 fs/jbd2/transaction.c:1260
__ext4_journal_get_write_access+0x1e6/0x6a0 fs/ext4/ext4_jbd2.c:239
ext4_reserve_inode_write+0x271/0x360 fs/ext4/inode.c:5731
__ext4_mark_inode_dirty+0x127/0x670 fs/ext4/inode.c:5905
ext4_dirty_inode+0xc9/0x110 fs/ext4/inode.c:5942
__mark_inode_dirty+0x1ef/0x610 fs/fs-writeback.c:2477
generic_update_time fs/inode.c:1907 [inline]
inode_update_time fs/inode.c:1920 [inline]
__file_update_time fs/inode.c:2109 [inline]
file_update_time+0x3d4/0x410 fs/inode.c:2139
ext4_page_mkwrite+0x200/0xde0 fs/ext4/inode.c:6060
do_page_mkwrite+0x18e/0x480 mm/memory.c:3091
wp_page_shared mm/memory.c:3478 [inline]
do_wp_page+0x137d/0x26d0 mm/memory.c:3628
handle_pte_fault+0x943/0xd90 mm/memory.c:5316
__handle_mm_fault+0xcdd/0x1230 mm/memory.c:5441
handle_mm_fault+0x3bd/0x880 mm/memory.c:5606
do_user_addr_fault+0xa8b/0x12c0 arch/x86/mm/fault.c:1362
handle_page_fault arch/x86/mm/fault.c:1505 [inline]
exc_page_fault+0x7f/0x120 arch/x86/mm/fault.c:1563
asm_exc_page_fault+0x26/0x30 arch/x86/include/asm/idtentry.h:623
RIP: 0033:0x7f3c2a7c5fde
RSP: 002b:00007ffeacb5c3f0 EFLAGS: 00010246
RAX: 00007f3c2864ec40 RBX: 00005579abd4c940 RCX: 000000000000acf8
RDX: 0000000000000000 RSI: 00005579abd4c940 RDI: 00007f3c286fdf98
RBP: 00005579abd4c970 R08: 00000000000fdf58 R09: 00000000000fdf58
R10: 0000000000000002 R11: 000000000000c9c8 R12: 0000000000000025
R13: 000000000004d650 R14: 0000000000000000 R15: 00007ffeacb5c410
</TASK>
INFO: task rs:main Q:Reg:7643 blocked for more than 143 seconds.
Not tainted 6.9.0-rc6 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
task:rs:main Q:Reg state:D stack:24696 pid:7643 tgid:7606 ppid:1
flags:0x00000000
Call Trace:
<TASK>
context_switch kernel/sched/core.c:5409 [inline]
__schedule+0xf19/0x15e0 kernel/sched/core.c:6746
__schedule_loop kernel/sched/core.c:6823 [inline]
schedule+0x147/0x310 kernel/sched/core.c:6838
io_schedule+0x87/0x100 kernel/sched/core.c:9044
bit_wait_io+0x12/0xc0 kernel/sched/wait_bit.c:209
__wait_on_bit+0xb0/0x320 kernel/sched/wait_bit.c:49
out_of_line_wait_on_bit+0x1c7/0x240 kernel/sched/wait_bit.c:64
wait_on_bit_io include/linux/wait_bit.h:101 [inline]
do_get_write_access+0x10c/0xcd0 fs/jbd2/transaction.c:1111
jbd2_journal_get_write_access+0x206/0x250 fs/jbd2/transaction.c:1260
__ext4_journal_get_write_access+0x1e6/0x6a0 fs/ext4/ext4_jbd2.c:239
ext4_reserve_inode_write+0x271/0x360 fs/ext4/inode.c:5731
__ext4_mark_inode_dirty+0x127/0x670 fs/ext4/inode.c:5905
ext4_dirty_inode+0xc9/0x110 fs/ext4/inode.c:5942
__mark_inode_dirty+0x1ef/0x610 fs/fs-writeback.c:2477
generic_update_time fs/inode.c:1907 [inline]
inode_update_time fs/inode.c:1920 [inline]
__file_update_time fs/inode.c:2109 [inline]
file_modified_flags+0x423/0x490 fs/inode.c:2180
ext4_write_checks+0x24a/0x2c0 fs/ext4/file.c:279
ext4_buffered_write_iter+0xa1/0x340 fs/ext4/file.c:295
ext4_file_write_iter+0x1d1/0x1a10
call_write_iter include/linux/fs.h:2110 [inline]
new_sync_write fs/read_write.c:497 [inline]
vfs_write+0xa61/0xcf0 fs/read_write.c:590
ksys_write+0x17b/0x2a0 fs/read_write.c:643
do_syscall_x64 arch/x86/entry/common.c:52 [inline]
do_syscall_64+0xe4/0x240 arch/x86/entry/common.c:83
entry_SYSCALL_64_after_hwframe+0x67/0x6f
RIP: 0033:0x7f09137c4fef
RSP: 002b:00007f09123ff860 EFLAGS: 00000293 ORIG_RAX: 0000000000000001
RAX: ffffffffffffffda RBX: 00007f0900024db0 RCX: 00007f09137c4fef
RDX: 000000000000057c RSI: 00007f0900025050 RDI: 0000000000000007
RBP: 000000000000057c R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000293 R12: 00007f0900025050
R13: 0000000000000000 R14: 0000000000000083 R15: 00007f0900024db0
</TASK>

Showing all locks held in the system:
4 locks held by kworker/1:0/24:
#0: ffff888013478948 ((wq_completion)events){+.+.}-{0:0}, at:
process_one_work kernel/workqueue.c:3229 [inline]
#0: ffff888013478948 ((wq_completion)events){+.+.}-{0:0}, at:
process_scheduled_works+0x94f/0x14a0 kernel/workqueue.c:3335
#1: ffffc900001bfd00
((work_completion)(&(&ap->scsi_rescan_task)->work)){+.+.}-{0:0}, at:
process_one_work kernel/workqueue.c:3230 [inline]
#1: ffffc900001bfd00
((work_completion)(&(&ap->scsi_rescan_task)->work)){+.+.}-{0:0}, at:
process_scheduled_works+0x979/0x14a0 kernel/workqueue.c:3335
#2: ffff88801c4a4360 (&ap->scsi_scan_mutex){+.+.}-{3:3}, at:
ata_scsi_dev_rescan+0x3e/0x3c0 drivers/ata/libata-scsi.c:4736
#3: ffff88801cd94378 (&dev->mutex){....}-{3:3}, at: device_lock
include/linux/device.h:990 [inline]
#3: ffff88801cd94378 (&dev->mutex){....}-{3:3}, at:
scsi_rescan_device+0x32/0x290 drivers/scsi/scsi_scan.c:1684
4 locks held by kworker/u10:1/30:
#0: ffff888060e8f148 ((wq_completion)writeback){+.+.}-{0:0}, at:
process_one_work kernel/workqueue.c:3229 [inline]
#0: ffff888060e8f148 ((wq_completion)writeback){+.+.}-{0:0}, at:
process_scheduled_works+0x94f/0x14a0 kernel/workqueue.c:3335
#1: ffffc9000083fd00
((work_completion)(&(&wb->dwork)->work)){+.+.}-{0:0}, at:
process_one_work kernel/workqueue.c:3230 [inline]
#1: ffffc9000083fd00
((work_completion)(&(&wb->dwork)->work)){+.+.}-{0:0}, at:
process_scheduled_works+0x979/0x14a0 kernel/workqueue.c:3335
#2: ffff88801966c0e0 (&type->s_umount_key#53){++++}-{3:3}, at:
super_trylock_shared+0x22/0xf0 fs/super.c:561
#3: ffff88801966ab98 (&sbi->s_writepages_rwsem){++++}-{0:0}, at:
ext4_writepages_down_read fs/ext4/ext4.h:1769 [inline]
#3: ffff88801966ab98 (&sbi->s_writepages_rwsem){++++}-{0:0}, at:
ext4_writepages+0x1be/0x3c0 fs/ext4/inode.c:2767
1 lock held by khungtaskd/33:
#0: ffffffff8dd343e0 (rcu_read_lock){....}-{1:2}, at:
rcu_lock_acquire include/linux/rcupdate.h:329 [inline]
#0: ffffffff8dd343e0 (rcu_read_lock){....}-{1:2}, at: rcu_read_lock
include/linux/rcupdate.h:781 [inline]
#0: ffffffff8dd343e0 (rcu_read_lock){....}-{1:2}, at:
debug_show_all_locks+0x54/0x2a0 kernel/locking/lockdep.c:6614
6 locks held by kworker/u10:2/53:
#0: ffff888060e8f148 ((wq_completion)writeback){+.+.}-{0:0}, at:
process_one_work kernel/workqueue.c:3229 [inline]
#0: ffff888060e8f148 ((wq_completion)writeback){+.+.}-{0:0}, at:
process_scheduled_works+0x94f/0x14a0 kernel/workqueue.c:3335
#1: ffffc900009afd00
((work_completion)(&(&wb->dwork)->work)){+.+.}-{0:0}, at:
process_one_work kernel/workqueue.c:3230 [inline]
#1: ffffc900009afd00
((work_completion)(&(&wb->dwork)->work)){+.+.}-{0:0}, at:
process_scheduled_works+0x979/0x14a0 kernel/workqueue.c:3335
#2: ffff88801966c0e0 (&type->s_umount_key#53){++++}-{3:3}, at:
super_trylock_shared+0x22/0xf0 fs/super.c:561
#3: ffff88801966ab98 (&sbi->s_writepages_rwsem){++++}-{0:0}, at:
ext4_writepages_down_read fs/ext4/ext4.h:1769 [inline]
#3: ffff88801966ab98 (&sbi->s_writepages_rwsem){++++}-{0:0}, at:
ext4_writepages+0x1be/0x3c0 fs/ext4/inode.c:2767
#4: ffff888019668950 (jbd2_handle){++++}-{0:0}, at:
start_this_handle+0x1f81/0x21e0 fs/jbd2/transaction.c:463
#5: ffff888077e08288 (&ei->i_data_sem){++++}-{3:3}, at:
ext4_map_blocks+0x9b2/0x1cb0 fs/ext4/inode.c:616
4 locks held by kworker/u10:5/2795:
#0: ffff888060e8f148 ((wq_completion)writeback){+.+.}-{0:0}, at:
process_one_work kernel/workqueue.c:3229 [inline]
#0: ffff888060e8f148 ((wq_completion)writeback){+.+.}-{0:0}, at:
process_scheduled_works+0x94f/0x14a0 kernel/workqueue.c:3335
#1: ffffc900096ffd00
((work_completion)(&(&wb->dwork)->work)){+.+.}-{0:0}, at:
process_one_work kernel/workqueue.c:3230 [inline]
#1: ffffc900096ffd00
((work_completion)(&(&wb->dwork)->work)){+.+.}-{0:0}, at:
process_scheduled_works+0x979/0x14a0 kernel/workqueue.c:3335
#2: ffff88801966c0e0 (&type->s_umount_key#53){++++}-{3:3}, at:
super_trylock_shared+0x22/0xf0 fs/super.c:561
#3: ffff88801966ab98 (&sbi->s_writepages_rwsem){++++}-{0:0}, at:
ext4_writepages_down_read fs/ext4/ext4.h:1769 [inline]
#3: ffff88801966ab98 (&sbi->s_writepages_rwsem){++++}-{0:0}, at:
ext4_writepages+0x1be/0x3c0 fs/ext4/inode.c:2767
3 locks held by systemd-journal/4529:
#0: ffff88806436c4a8 (&vma->vm_lock->lock){++++}-{3:3}, at:
vma_start_read include/linux/mm.h:677 [inline]
#0: ffff88806436c4a8 (&vma->vm_lock->lock){++++}-{3:3}, at:
lock_vma_under_rcu+0x2f7/0x730 mm/memory.c:5762
#1: ffff88801966c518 (sb_pagefaults){.+.+}-{0:0}, at:
__sb_start_write include/linux/fs.h:1664 [inline]
#1: ffff88801966c518 (sb_pagefaults){.+.+}-{0:0}, at:
sb_start_pagefault include/linux/fs.h:1829 [inline]
#1: ffff88801966c518 (sb_pagefaults){.+.+}-{0:0}, at:
ext4_page_mkwrite+0x1e9/0xde0 fs/ext4/inode.c:6059
#2: ffff888019668950 (jbd2_handle){++++}-{0:0}, at:
start_this_handle+0x1f81/0x21e0 fs/jbd2/transaction.c:463
1 lock held by in:imklog/7641:
4 locks held by rs:main Q:Reg/7643:
#0: ffff888068c299c8 (&f->f_pos_lock){+.+.}-{3:3}, at:
__fdget_pos+0x255/0x320 fs/file.c:1191
#1: ffff88801966c420 (sb_writers#4){.+.+}-{0:0}, at: file_start_write
include/linux/fs.h:2855 [inline]
#1: ffff88801966c420 (sb_writers#4){.+.+}-{0:0}, at:
vfs_write+0x230/0xcf0 fs/read_write.c:586
#2: ffff888077cc9800 (&sb->s_type->i_mutex_key#7){++++}-{3:3}, at:
inode_lock include/linux/fs.h:795 [inline]
#2: ffff888077cc9800 (&sb->s_type->i_mutex_key#7){++++}-{3:3}, at:
ext4_buffered_write_iter+0x96/0x340 fs/ext4/file.c:294
#3: ffff888019668950 (jbd2_handle){++++}-{0:0}, at:
start_this_handle+0x1f81/0x21e0 fs/jbd2/transaction.c:463

=============================================

NMI backtrace for cpu 1
CPU: 1 PID: 33 Comm: khungtaskd Not tainted 6.9.0-rc6 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS
1.13.0-1ubuntu1.1 04/01/2014
Call Trace:
<TASK>
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0x201/0x300 lib/dump_stack.c:114
nmi_cpu_backtrace+0x41e/0x450 lib/nmi_backtrace.c:113
nmi_trigger_cpumask_backtrace+0x185/0x300 lib/nmi_backtrace.c:62
trigger_all_cpu_backtrace include/linux/nmi.h:160 [inline]
check_hung_uninterruptible_tasks kernel/hung_task.c:223 [inline]
watchdog+0xddb/0xe20 kernel/hung_task.c:380
kthread+0x2ed/0x390 kernel/kthread.c:388
ret_from_fork+0x4b/0x80 arch/x86/kernel/process.c:147
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:244
</TASK>
Sending NMI from CPU 1 to CPUs 0:
NMI backtrace for cpu 0
CPU: 0 PID: 31 Comm: kworker/u9:1 Not tainted 6.9.0-rc6 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS
1.13.0-1ubuntu1.1 04/01/2014
Workqueue: events_unbound toggle_allocation_gate
RIP: 0010:div_u64_rem include/linux/math64.h:29 [inline]
RIP: 0010:div_u64 include/linux/math64.h:130 [inline]
RIP: 0010:___update_load_avg kernel/sched/pelt.c:265 [inline]
RIP: 0010:__update_load_avg_cfs_rq+0x5c9/0xa30 kernel/sched/pelt.c:327
Code: 00 00 48 89 f8 48 c1 e8 03 80 3c 30 00 74 0f e8 9d d0 87 00 48
be 00 00 00 00 00 fc ff df 49 8b 86 90 00 00 00 31 d2 48 f7 f5 <48> 89
c3 49 8d be a8 00 00 00 48 89 f8 48 c1 e8 03 80 3c 30 00 74
RSP: 0000:ffffc9000084f670 EFLAGS: 00000046
RAX: 00000000000001de RBX: 00000000000001f7 RCX: 0000000000001400
RDX: 0000000000005d47 RSI: dffffc0000000000 RDI: ffff88804403e790
RBP: 000000000000b7e8 R08: 0000000000000001 R09: 000000000000056a
R10: 000000000000016a R11: fffffbfff1e8f856 R12: 0000000000000000
R13: 0000000000067c00 R14: ffff88804403e700 R15: 000000000000b7e8
FS: 0000000000000000(0000) GS:ffff888044000000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0904059118 CR3: 000000000db34000 CR4: 0000000000750ef0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
<NMI>
</NMI>
<TASK>
update_cfs_rq_load_avg+0x483/0x570 kernel/sched/fair.c:4640
update_load_avg+0x193/0x1400 kernel/sched/fair.c:4753
dequeue_entity+0x8d/0xc00 kernel/sched/fair.c:5384
dequeue_task_fair+0x18e/0x990 kernel/sched/fair.c:6821
deactivate_task kernel/sched/core.c:2157 [inline]
__schedule+0x50f/0x15e0 kernel/sched/core.c:6692
__schedule_loop kernel/sched/core.c:6823 [inline]
schedule+0x147/0x310 kernel/sched/core.c:6838
toggle_allocation_gate+0x15b/0x240 mm/kfence/core.c:828
process_one_work kernel/workqueue.c:3254 [inline]
process_scheduled_works+0x9c9/0x14a0 kernel/workqueue.c:3335
worker_thread+0x85c/0xd50 kernel/workqueue.c:3416
kthread+0x2ed/0x390 kernel/kthread.c:388
ret_from_fork+0x4b/0x80 arch/x86/kernel/process.c:147
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:244
</TASK>
INFO: NMI handler (nmi_cpu_backtrace_handler) took too long to run: 1.097 msecs
```
If you have any questions, please contact us.

Reported by Yue Sun <samsun1006219@xxxxxxxxx>
Reported by xingwei lee <xrivendell7@xxxxxxxxx>

Best Regards,
Yue

Attachment: config
Description: Binary data

#define _GNU_SOURCE

#include <dirent.h>
#include <endian.h>
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/prctl.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>

static unsigned long long procid;

static void sleep_ms(uint64_t ms)
{
usleep(ms * 1000);
}

static uint64_t current_time_ms(void)
{
struct timespec ts;
if (clock_gettime(CLOCK_MONOTONIC, &ts))
exit(1);
return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000;
}

static bool write_file(const char* file, const char* what, ...)
{
char buf[1024];
va_list args;
va_start(args, what);
vsnprintf(buf, sizeof(buf), what, args);
va_end(args);
buf[sizeof(buf) - 1] = 0;
int len = strlen(buf);
int fd = open(file, O_WRONLY | O_CLOEXEC);
if (fd == -1)
return false;
if (write(fd, buf, len) != len) {
int err = errno;
close(fd);
errno = err;
return false;
}
close(fd);
return true;
}

static long syz_mod_param(volatile long a0, volatile long a1, volatile long a2,
volatile long a3, volatile long a4)
{
int fd, sysfd;
char buf[1024], sysbuf[1024], input[1024];
char* hash;
strncpy(buf, (char*)a0, sizeof(buf) - 1);
buf[sizeof(buf) - 1] = 0;
while ((hash = strchr(buf, '#'))) {
*hash = '0' + (char)(a1 % 10);
a1 /= 10;
}
fd = open(buf, a2, 0);
strncpy(sysbuf, (char*)a3, sizeof(sysbuf) - 1);
sysbuf[sizeof(sysbuf) - 1] = 0;
sysfd = open(sysbuf, O_RDWR, 0);
strncpy(input, (char*)a4, sizeof(input) - 1);
hash = strchr(input, '\0');
sysfd = write(sysfd, input, hash - input + 1);
return fd;
}

static void kill_and_wait(int pid, int* status)
{
kill(-pid, SIGKILL);
kill(pid, SIGKILL);
for (int i = 0; i < 100; i++) {
if (waitpid(-1, status, WNOHANG | __WALL) == pid)
return;
usleep(1000);
}
DIR* dir = opendir("/sys/fs/fuse/connections");
if (dir) {
for (;;) {
struct dirent* ent = readdir(dir);
if (!ent)
break;
if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0)
continue;
char abort[300];
snprintf(abort, sizeof(abort), "/sys/fs/fuse/connections/%s/abort",
ent->d_name);
int fd = open(abort, O_WRONLY);
if (fd == -1) {
continue;
}
if (write(fd, abort, 1) < 0) {
}
close(fd);
}
closedir(dir);
} else {
}
while (waitpid(-1, status, __WALL) != pid) {
}
}

static void setup_test()
{
prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
setpgrp();
write_file("/proc/self/oom_score_adj", "1000");
}

static void execute_one(void);

#define WAIT_FLAGS __WALL

static void loop(void)
{
int iter = 0;
for (;; iter++) {
int pid = fork();
if (pid < 0)
exit(1);
if (pid == 0) {
setup_test();
execute_one();
exit(0);
}
int status = 0;
uint64_t start = current_time_ms();
for (;;) {
if (waitpid(-1, &status, WNOHANG | WAIT_FLAGS) == pid)
break;
sleep_ms(1);
if (current_time_ms() - start < 5000)
continue;
kill_and_wait(pid, &status);
break;
}
}
}

uint64_t r[1] = {0xffffffffffffffff};

void execute_one(void)
{
intptr_t res = 0;
memcpy((void*)0x20000000, "/dev/sg#\000", 9);
memcpy((void*)0x20000080, "/sys/module/sg/parameters/allow_dio\000", 36);
memcpy((void*)0x20000040, "100\000", 4);
res = -1;
res = syz_mod_param(/*dev=*/0x20000000, /*id=*/0, /*flags=*/0,
/*file=*/0x20000080, /*buf=*/0x20000040);
if (res != -1)
r[0] = res;
memcpy((void*)0x20000140,
"\x02\x00\x00\x00\x00\x00\x00\x00\xa1\x09\x01\xa1\x04\x4a\x76\x68\xd2"
"\x25",
18);
syscall(__NR_ioctl, /*fd=*/r[0], /*cmd=*/1, /*arg=*/0x20000140ul);
}
int main(void)
{
syscall(__NR_mmap, /*addr=*/0x1ffff000ul, /*len=*/0x1000ul, /*prot=*/0ul,
/*flags=MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE*/ 0x32ul, /*fd=*/-1,
/*offset=*/0ul);
syscall(__NR_mmap, /*addr=*/0x20000000ul, /*len=*/0x1000000ul,
/*prot=PROT_WRITE|PROT_READ|PROT_EXEC*/ 7ul,
/*flags=MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE*/ 0x32ul, /*fd=*/-1,
/*offset=*/0ul);
syscall(__NR_mmap, /*addr=*/0x21000000ul, /*len=*/0x1000ul, /*prot=*/0ul,
/*flags=MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE*/ 0x32ul, /*fd=*/-1,
/*offset=*/0ul);
for (procid = 0; procid < 4; procid++) {
if (fork() == 0) {
loop();
}
}
sleep(1000000);
return 0;
}

Next message: Chengming Zhou: "[PATCH v2 0/2] mm/ksm: fix some accounting problems"
Previous message: Nicolin Chen: "Re: [PATCH RFCv1 03/14] iommufd: Prepare for viommu structures and functions"
Next in thread: Hillf Danton: "Re: [Linux kernel bug] INFO: task hung in blk_mq_get_tag"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]