Re: INFO: task hung in lock_sock_nested (2)
From: Stefano Garzarella
Date: Mon Feb 24 2020 - 05:09:13 EST
On Sun, Feb 23, 2020 at 03:50:25PM +0800, Hillf Danton wrote:
> On Sat, 22 Feb 2020 10:58:12 -0800
> > syzbot found the following crash on:
> >
> > HEAD commit: 2bb07f4e tc-testing: updated tdc tests for basic filter
> > git tree: net-next
> > console output: https://syzkaller.appspot.com/x/log.txt?x=122efdede00000
> > kernel config: https://syzkaller.appspot.com/x/.config?x=768cc3d3e277cc16
> > dashboard link: https://syzkaller.appspot.com/bug?extid=731710996d79d0d58fbc
> > compiler: gcc (GCC) 9.0.0 20181231 (experimental)
> > syz repro: https://syzkaller.appspot.com/x/repro.syz?x=14887de9e00000
> > C reproducer: https://syzkaller.appspot.com/x/repro.c?x=149eec81e00000
> >
> > The bug was bisected to:
> >
> > commit 408624af4c89989117bb2c6517bd50b7708a2fcd
> > Author: Stefano Garzarella <sgarzare@xxxxxxxxxx>
> > Date: Tue Dec 10 10:43:06 2019 +0000
> >
> > vsock: use local transport when it is loaded
> >
> > bisection log: https://syzkaller.appspot.com/x/bisect.txt?x=1011e27ee00000
> > final crash: https://syzkaller.appspot.com/x/report.txt?x=1211e27ee00000
> > console output: https://syzkaller.appspot.com/x/log.txt?x=1411e27ee00000
> >
> > IMPORTANT: if you fix the bug, please add the following tag to the commit:
> > Reported-by: syzbot+731710996d79d0d58fbc@xxxxxxxxxxxxxxxxxxxxxxxxx
> > Fixes: 408624af4c89 ("vsock: use local transport when it is loaded")
> >
> > INFO: task syz-executor280:9768 blocked for more than 143 seconds.
> > Not tainted 5.6.0-rc1-syzkaller #0
> > "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
> > syz-executor280 D27912 9768 9766 0x00000000
> > Call Trace:
> > context_switch kernel/sched/core.c:3386 [inline]
> > __schedule+0x934/0x1f90 kernel/sched/core.c:4082
> > schedule+0xdc/0x2b0 kernel/sched/core.c:4156
> > __lock_sock+0x165/0x290 net/core/sock.c:2413
> > lock_sock_nested+0xfe/0x120 net/core/sock.c:2938
> > virtio_transport_release+0xc4/0xd60 net/vmw_vsock/virtio_transport_common.c:832
> > vsock_assign_transport+0xf3/0x3b0 net/vmw_vsock/af_vsock.c:454
> > vsock_stream_connect+0x2b3/0xc70 net/vmw_vsock/af_vsock.c:1288
> > __sys_connect_file+0x161/0x1c0 net/socket.c:1857
> > __sys_connect+0x174/0x1b0 net/socket.c:1874
> > __do_sys_connect net/socket.c:1885 [inline]
> > __se_sys_connect net/socket.c:1882 [inline]
> > __x64_sys_connect+0x73/0xb0 net/socket.c:1882
> > do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294
> > entry_SYSCALL_64_after_hwframe+0x49/0xbe
> > RIP: 0033:0x440209
> > Code: Bad RIP value.
> > RSP: 002b:00007ffdb9f67718 EFLAGS: 00000246 ORIG_RAX: 000000000000002a
> > RAX: ffffffffffffffda RBX: 00000000004002c8 RCX: 0000000000440209
> > RDX: 0000000000000010 RSI: 0000000020000440 RDI: 0000000000000003
> > RBP: 00000000006ca018 R08: 00000000004002c8 R09: 00000000004002c8
> > R10: 00000000004002c8 R11: 0000000000000246 R12: 0000000000401a90
> > R13: 0000000000401b20 R14: 0000000000000000 R15: 0000000000000000
> >
> > Showing all locks held in the system:
> > 1 lock held by khungtaskd/951:
> > #0: ffffffff89bac240 (rcu_read_lock){....}, at: debug_show_all_locks+0x5f/0x279 kernel/locking/lockdep.c:5333
> > 1 lock held by rsyslogd/9652:
> > #0: ffff8880a6533120 (&f->f_pos_lock){+.+.}, at: __fdget_pos+0xee/0x110 fs/file.c:821
> > 2 locks held by getty/9742:
> > #0: ffff8880a693f090 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x33/0x40 drivers/tty/tty_ldsem.c:340
> > #1: ffffc900061bb2e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x220/0x1bf0 drivers/tty/n_tty.c:2156
> > 2 locks held by getty/9743:
> > #0: ffff88809f7a1090 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x33/0x40 drivers/tty/tty_ldsem.c:340
> > #1: ffffc900061b72e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x220/0x1bf0 drivers/tty/n_tty.c:2156
> > 2 locks held by getty/9744:
> > #0: ffff88809be3e090 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x33/0x40 drivers/tty/tty_ldsem.c:340
> > #1: ffffc900061632e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x220/0x1bf0 drivers/tty/n_tty.c:2156
> > 2 locks held by getty/9745:
> > #0: ffff88808eb1e090 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x33/0x40 drivers/tty/tty_ldsem.c:340
> > #1: ffffc900061bf2e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x220/0x1bf0 drivers/tty/n_tty.c:2156
> > 2 locks held by getty/9746:
> > #0: ffff88808d33a090 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x33/0x40 drivers/tty/tty_ldsem.c:340
> > #1: ffffc900061732e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x220/0x1bf0 drivers/tty/n_tty.c:2156
> > 2 locks held by getty/9747:
> > #0: ffff8880a6a0c090 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x33/0x40 drivers/tty/tty_ldsem.c:340
> > #1: ffffc900061c32e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x220/0x1bf0 drivers/tty/n_tty.c:2156
> > 2 locks held by getty/9748:
> > #0: ffff8880a6e4d090 (&tty->ldisc_sem){++++}, at: ldsem_down_read+0x33/0x40 drivers/tty/tty_ldsem.c:340
> > #1: ffffc900061332e0 (&ldata->atomic_read_lock){+.+.}, at: n_tty_read+0x220/0x1bf0 drivers/tty/n_tty.c:2156
> > 1 lock held by syz-executor280/9768:
> > #0: ffff8880987cb8d0 (sk_lock-AF_VSOCK){+.+.}, at: lock_sock include/net/sock.h:1516 [inline]
> > #0: ffff8880987cb8d0 (sk_lock-AF_VSOCK){+.+.}, at: vsock_stream_connect+0xfb/0xc70 net/vmw_vsock/af_vsock.c:1258
> >
> > =============================================
> >
> > NMI backtrace for cpu 1
> > CPU: 1 PID: 951 Comm: khungtaskd Not tainted 5.6.0-rc1-syzkaller #0
> > Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
> > Call Trace:
> > __dump_stack lib/dump_stack.c:77 [inline]
> > dump_stack+0x197/0x210 lib/dump_stack.c:118
> > nmi_cpu_backtrace.cold+0x70/0xb2 lib/nmi_backtrace.c:101
> > nmi_trigger_cpumask_backtrace+0x23b/0x28b lib/nmi_backtrace.c:62
> > arch_trigger_cpumask_backtrace+0x14/0x20 arch/x86/kernel/apic/hw_nmi.c:38
> > trigger_all_cpu_backtrace include/linux/nmi.h:146 [inline]
> > check_hung_uninterruptible_tasks kernel/hung_task.c:205 [inline]
> > watchdog+0xb11/0x10c0 kernel/hung_task.c:289
> > kthread+0x361/0x430 kernel/kthread.c:255
> > ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352
> > Sending NMI from CPU 1 to CPUs 0:
> > NMI backtrace for cpu 0
> > CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.6.0-rc1-syzkaller #0
> > Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
> > RIP: 0010:group_is_overloaded kernel/sched/fair.c:7929 [inline]
> > RIP: 0010:group_classify kernel/sched/fair.c:7964 [inline]
> > RIP: 0010:update_sg_lb_stats kernel/sched/fair.c:8077 [inline]
> > RIP: 0010:update_sd_lb_stats kernel/sched/fair.c:8565 [inline]
> > RIP: 0010:find_busiest_group+0xa33/0x3250 kernel/sched/fair.c:8793
> > Code: 89 f8 83 e0 07 83 c0 03 40 38 f0 7c 09 40 84 f6 0f 85 f7 1f 00 00 48 8b b5 c0 fd ff ff 41 8b 41 2c 48 c1 ee 03 42 0f b6 34 26 <40> 84 f6 74 0a 40 80 fe 03 0f 8e 8f 1f 00 00 44 8b 6b 20 44 39 ea
> > RSP: 0018:ffffc90000007850 EFLAGS: 00000a06
> > RAX: 000000000000006e RBX: ffffc90000007938 RCX: 00000000000003fa
> > RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff8880a9a8282c
> > RBP: ffffc90000007af0 R08: ffffffff89a7a440 R09: ffff8880a9a82800
> > R10: 0000000000000000 R11: ffff8880a9a83f27 R12: dffffc0000000000
> > R13: ffff8880a9a83e80 R14: ffffc90000007ac8 R15: ffffc90000007c08
> > FS: 0000000000000000(0000) GS:ffff8880ae800000(0000) knlGS:0000000000000000
> > CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> > CR2: ffffffffff600400 CR3: 000000009fde0000 CR4: 00000000001406f0
> > DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
> > DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
> > Call Trace:
> > <IRQ>
> > load_balance+0x38c/0x2b50 kernel/sched/fair.c:9161
> > rebalance_domains+0x739/0x1000 kernel/sched/fair.c:9588
> > _nohz_idle_balance+0x336/0x3f0 kernel/sched/fair.c:10002
> > nohz_idle_balance kernel/sched/fair.c:10048 [inline]
> > run_rebalance_domains+0x1c6/0x2d0 kernel/sched/fair.c:10237
> > __do_softirq+0x262/0x98c kernel/softirq.c:292
> > invoke_softirq kernel/softirq.c:373 [inline]
> > irq_exit+0x19b/0x1e0 kernel/softirq.c:413
> > scheduler_ipi+0x38c/0x610 kernel/sched/core.c:2349
> > smp_reschedule_interrupt+0x78/0x4c0 arch/x86/kernel/smp.c:244
> > reschedule_interrupt+0xf/0x20 arch/x86/entry/entry_64.S:853
> > </IRQ>
> > RIP: 0010:native_safe_halt+0xe/0x10 arch/x86/include/asm/irqflags.h:61
> > Code: 58 f5 c3 f9 eb 8a cc cc cc cc cc cc e9 07 00 00 00 0f 00 2d 74 40 58 00 f4 c3 66 90 e9 07 00 00 00 0f 00 2d 64 40 58 00 fb f4 <c3> cc 55 48 89 e5 41 57 41 56 41 55 41 54 53 e8 ce ac 72 f9 e8 e9
> > RSP: 0018:ffffffff89a07ce8 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff02
> > RAX: 1ffffffff136761a RBX: ffffffff89a7a440 RCX: 0000000000000000
> > RDX: dffffc0000000000 RSI: 0000000000000006 RDI: ffffffff89a7acd4
> > RBP: ffffffff89a07d18 R08: ffffffff89a7a440 R09: 0000000000000000
> > R10: 0000000000000000 R11: 0000000000000000 R12: dffffc0000000000
> > R13: ffffffff8aa5ab80 R14: 0000000000000000 R15: 0000000000000000
> > arch_cpu_idle+0xa/0x10 arch/x86/kernel/process.c:686
> > default_idle_call+0x84/0xb0 kernel/sched/idle.c:94
> > cpuidle_idle_call kernel/sched/idle.c:154 [inline]
> > do_idle+0x3c8/0x6e0 kernel/sched/idle.c:269
> > cpu_startup_entry+0x1b/0x20 kernel/sched/idle.c:361
> > rest_init+0x23b/0x371 init/main.c:654
> > arch_call_rest_init+0xe/0x1b
> > start_kernel+0x886/0x8c5 init/main.c:992
> > x86_64_start_reservations+0x29/0x2b arch/x86/kernel/head64.c:490
> > x86_64_start_kernel+0x77/0x7b arch/x86/kernel/head64.c:471
> > secondary_startup_64+0xa4/0xb0 arch/x86/kernel/head_64.S:242
> >
> >
> > ---
> > This bug is generated by a bot. It may contain errors.
> > See https://goo.gl/tpsmEJ for more information about syzbot.
> > syzbot engineers can be reached at syzkaller@xxxxxxxxxxxxxxxxx
> >
> > syzbot will keep track of this bug report. See:
> > https://goo.gl/tpsmEJ#status for how to communicate with syzbot.
> > For information about bisection process see: https://goo.gl/tpsmEJ#bisection
> > syzbot can test patches for this bug, for details see:
> > https://goo.gl/tpsmEJ#testing-patches
>
>
> Seems like vsock needs a word to track lock owner in an attempt to
> avoid trying to lock sock while the current is the lock owner.
Thanks for this possible solution.
What about using sock_owned_by_user()?
We should fix also hyperv_transport, because it could suffer from the same
problem.
At this point, it might be better to call vsk->transport->release(vsk)
always with the lock taken and remove it in the transports as in the
following patch.
What do you think?
diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
index 9c5b2a91baad..a073d8efca33 100644
--- a/net/vmw_vsock/af_vsock.c
+++ b/net/vmw_vsock/af_vsock.c
@@ -753,20 +753,18 @@ static void __vsock_release(struct sock *sk, int level)
vsk = vsock_sk(sk);
pending = NULL; /* Compiler warning. */
- /* The release call is supposed to use lock_sock_nested()
- * rather than lock_sock(), if a sock lock should be acquired.
- */
- if (vsk->transport)
- vsk->transport->release(vsk);
- else if (sk->sk_type == SOCK_STREAM)
- vsock_remove_sock(vsk);
-
/* When "level" is SINGLE_DEPTH_NESTING, use the nested
* version to avoid the warning "possible recursive locking
* detected". When "level" is 0, lock_sock_nested(sk, level)
* is the same as lock_sock(sk).
*/
lock_sock_nested(sk, level);
+
+ if (vsk->transport)
+ vsk->transport->release(vsk);
+ else if (sk->sk_type == SOCK_STREAM)
+ vsock_remove_sock(vsk);
+
sock_orphan(sk);
sk->sk_shutdown = SHUTDOWN_MASK;
diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c
index 3492c021925f..510f25f4a856 100644
--- a/net/vmw_vsock/hyperv_transport.c
+++ b/net/vmw_vsock/hyperv_transport.c
@@ -529,9 +529,7 @@ static void hvs_release(struct vsock_sock *vsk)
struct sock *sk = sk_vsock(vsk);
bool remove_sock;
- lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
remove_sock = hvs_close_lock_held(vsk);
- release_sock(sk);
if (remove_sock)
vsock_remove_sock(vsk);
}
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index d9f0c9c5425a..f3c4bab2f737 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -829,7 +829,6 @@ void virtio_transport_release(struct vsock_sock *vsk)
struct sock *sk = &vsk->sk;
bool remove_sock = true;
- lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
if (sk->sk_type == SOCK_STREAM)
remove_sock = virtio_transport_close(vsk);
@@ -837,7 +836,6 @@ void virtio_transport_release(struct vsock_sock *vsk)
list_del(&pkt->list);
virtio_transport_free_pkt(pkt);
}
- release_sock(sk);
if (remove_sock)
vsock_remove_sock(vsk);
Thanks,
Stefano