tip.today - scheduler bam boom crash (cpu hotplug)
From: Mike Galbraith
Date: Thu Jan 19 2017 - 02:32:42 EST
Mindless testing only, too sick to work, not sick enough to be immune
to boredom. Was verifying first warning wasn't somehow rt inspired,
but while doing so, plain nopreempt (and no rt patch set) went boom.
[ 203.088255] smpboot: CPU 1 is now offline
[ 203.168181] smpboot: CPU 2 is now offline
[ 203.221461] x86: Booting SMP configuration:
[ 203.221464] smpboot: Booting Node 0 Processor 1 APIC 0x2
[ 203.221728] ------------[ cut here ]------------
[ 203.221733] WARNING: CPU: 1 PID: 0 at kernel/sched/clock.c:149 set_sched_clock_stable+0x43/0x50
[ 203.221733] Modules linked in: nls_utf8(E) isofs(E) ebtable_filter(E) ebtables(E) fuse(E) nf_log_ipv6(E) xt_pkttype(E) xt_physdev(E) br_netfilter(E) nf_log_ipv4(E) nf_log_common(E) xt_LOG(E) xt_limit(E) af_packet(E) bridge(E) stp(E) llc(E) iscsi_ibft(E) iscsi_boot_sysfs(E) ip6t_REJECT(E) xt_tcpudp(E) nf_conntrack_ipv6(E) nf_defrag_ipv6(E) ip6table_raw(E) ipt_REJECT(E) iptable_raw(E) xt_CT(E) iptable_filter(E) ip6table_mangle(E) nf_conntrack_netbios_ns(E) nf_conntrack_broadcast(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) ip_tables(E) xt_conntrack(E) nf_conntrack(E) ip6table_filter(E) ip6_tables(E) x_tables(E) nls_iso8859_1(E) snd_hda_codec_hdmi(E) nls_cp437(E) intel_rapl(E) x86_pkg_temp_thermal(E) intel_powerclamp(E) coretemp(E) kvm_intel(E) snd_hda_codec_realtek(E) snd_hda_codec_generic(E) kvm(E) snd_hda_intel(E)
[ 203.221748] snd_hda_codec(E) irqbypass(E) crct10dif_pclmul(E) snd_hda_core(E) snd_hwdep(E) nfsd(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) pcbc(E) snd_pcm(E) auth_rpcgss(E) aesni_intel(E) aes_x86_64(E) snd_timer(E) nfs_acl(E) joydev(E) crypto_simd(E) snd(E) lockd(E) grace(E) iTCO_wdt(E) iTCO_vendor_support(E) lpc_ich(E) mei_me(E) i2c_i801(E) mei(E) pcspkr(E) glue_helper(E) mfd_core(E) shpchp(E) intel_smartconnect(E) sunrpc(E) soundcore(E) tpm_infineon(E) fan(E) thermal(E) battery(E) cryptd(E) efivarfs(E) sr_mod(E) cdrom(E) hid_logitech_hidpp(E) hid_logitech_dj(E) uas(E) usb_storage(E) hid_generic(E) usbhid(E) nouveau(E) wmi(E) i2c_algo_bit(E) drm_kms_helper(E) ahci(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) libahci(E) xhci_pci(E) ehci_pci(E) xhci_hcd(E) ehci_hcd(E)
[ 203.221765] ttm(E) libata(E) r8169(E) mii(E) drm(E) usbcore(E) fjes(E) video(E) button(E) sd_mod(E) vfat(E) fat(E) ext4(E) crc16(E) jbd2(E) mbcache(E) dm_mod(E) loop(E) sg(E) scsi_mod(E) autofs4(E)
[ 203.221773] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G E 4.10.0-tip-default #29
[ 203.221774] Hardware name: MEDION MS-7848/MS-7848, BIOS M7848W08.20C 09/23/2013
[ 203.221774] Call Trace:
[ 203.221778] dump_stack+0x63/0x90
[ 203.221780] __warn+0xd1/0xf0
[ 203.221782] warn_slowpath_null+0x1d/0x20
[ 203.221782] set_sched_clock_stable+0x43/0x50
[ 203.221784] early_init_intel+0x225/0x360
[ 203.221785] init_intel+0x18/0x2d0
[ 203.221786] identify_cpu+0x2d1/0x4d0
[ 203.221786] identify_secondary_cpu+0x18/0x80
[ 203.221789] smp_store_cpu_info+0x3e/0x40
[ 203.221790] start_secondary+0x53/0x180
[ 203.221791] start_cpu+0x14/0x14
[ 203.221792] ---[ end trace 262c7e4b746d5a76 ]---
....
[ 207.525918] smpboot: CPU 2 is now offline
[ 207.586516] smpboot: CPU 4 is now offline
[ 207.642988] smpboot: CPU 6 is now offline
[ 207.682207] x86: Booting SMP configuration:
[ 207.682210] smpboot: Booting Node 0 Processor 1 APIC 0x2
[ 207.682505] sched_clock: Marking stable (207412639708, 0)->(207410993286, 1646422)
[ 207.706220] smpboot: Booting Node 0 Processor 2 APIC 0x4
[ 207.706502] sched_clock: Marking stable (207524564558, 0)->(207522917721, 1646837)
[ 207.730376] smpboot: Booting Node 0 Processor 4 APIC 0x1
[ 207.730644] sched_clock: Marking stable (207585448402, 0)->(207583801309, 1647093)
[ 207.754593] smpboot: Booting Node 0 Processor 6 APIC 0x5
[ 207.754881] sched_clock: Marking stable (207641939733, 0)->(207640292394, 1647339)
[ 207.802195] smpboot: CPU 3 is now offline
[ 207.862396] smpboot: CPU 4 is now offline
[ 207.927159] ------------[ cut here ]------------
[ 207.927163] WARNING: CPU: 6 PID: 45 at kernel/sched/sched.h:807 assert_clock_updated.isra.62.part.63+0x25/0x27
[ 207.927164] rq->clock_update_flags < RQCF_ACT_SKIP
[ 207.927164] Modules linked in: nls_utf8(E) isofs(E) ebtable_filter(E) ebtables(E) fuse(E) nf_log_ipv6(E) xt_pkttype(E) xt_physdev(E) br_netfilter(E) nf_log_ipv4(E) nf_log_common(E) xt_LOG(E) xt_limit(E) af_packet(E) bridge(E) stp(E) llc(E) iscsi_ibft(E) iscsi_boot_sysfs(E) ip6t_REJECT(E) xt_tcpudp(E) nf_conntrack_ipv6(E) nf_defrag_ipv6(E) ip6table_raw(E) ipt_REJECT(E) iptable_raw(E) xt_CT(E) iptable_filter(E) ip6table_mangle(E) nf_conntrack_netbios_ns(E) nf_conntrack_broadcast(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) ip_tables(E) xt_conntrack(E) nf_conntrack(E) ip6table_filter(E) ip6_tables(E) x_tables(E) nls_iso8859_1(E) snd_hda_codec_hdmi(E) nls_cp437(E) intel_rapl(E) x86_pkg_temp_thermal(E) intel_powerclamp(E) coretemp(E) kvm_intel(E) snd_hda_codec_realtek(E) snd_hda_codec_generic(E) kvm(E) snd_hda_intel(E)
[ 207.927186] snd_hda_codec(E) irqbypass(E) crct10dif_pclmul(E) snd_hda_core(E) snd_hwdep(E) nfsd(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) pcbc(E) snd_pcm(E) auth_rpcgss(E) aesni_intel(E) aes_x86_64(E) snd_timer(E) nfs_acl(E) joydev(E) crypto_simd(E) snd(E) lockd(E) grace(E) iTCO_wdt(E) iTCO_vendor_support(E) lpc_ich(E) mei_me(E) i2c_i801(E) mei(E) pcspkr(E) glue_helper(E) mfd_core(E) shpchp(E) intel_smartconnect(E) sunrpc(E) soundcore(E) tpm_infineon(E) fan(E) thermal(E) battery(E) cryptd(E) efivarfs(E) sr_mod(E) cdrom(E) hid_logitech_hidpp(E) hid_logitech_dj(E) uas(E) usb_storage(E) hid_generic(E) usbhid(E) nouveau(E) wmi(E) i2c_algo_bit(E) drm_kms_helper(E) ahci(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) libahci(E) xhci_pci(E) ehci_pci(E) xhci_hcd(E) ehci_hcd(E)
[ 207.927212] ttm(E) libata(E) r8169(E) mii(E) drm(E) usbcore(E) fjes(E) video(E) button(E) sd_mod(E) vfat(E) fat(E) ext4(E) crc16(E) jbd2(E) mbcache(E) dm_mod(E) loop(E) sg(E) scsi_mod(E) autofs4(E)
[ 207.927230] CPU: 6 PID: 45 Comm: migration/6 Tainted: G W E 4.10.0-tip-default #29
[ 207.927231] Hardware name: MEDION MS-7848/MS-7848, BIOS M7848W08.20C 09/23/2013
[ 207.927231] Call Trace:
[ 207.927235] dump_stack+0x63/0x90
[ 207.927238] __warn+0xd1/0xf0
[ 207.927240] ? load_balance+0xa10/0xa10
[ 207.927242] warn_slowpath_fmt+0x4f/0x60
[ 207.927244] ? find_next_bit+0x19/0x20
[ 207.927245] ? cpumask_next_and+0x35/0x50
[ 207.927246] assert_clock_updated.isra.62.part.63+0x25/0x27
[ 207.927247] update_load_avg+0x855/0x950
[ 207.927249] ? load_balance+0xa10/0xa10
[ 207.927250] set_next_entity+0xa6/0x210
[ 207.927252] ? load_balance+0xa10/0xa10
[ 207.927252] pick_next_task_fair+0x78/0x550
[ 207.927255] ? sched_clock+0x9/0x10
[ 207.927256] ? sched_clock_cpu+0x11/0xc0
[ 207.927257] ? load_balance+0xa10/0xa10
[ 207.927258] sched_cpu_dying+0x251/0x2a0
[ 207.927260] ? fini_debug_store_on_cpu+0x34/0x40
[ 207.927261] ? sched_cpu_starting+0x60/0x60
[ 207.927263] cpuhp_invoke_callback+0x90/0x440
[ 207.927265] take_cpu_down+0x5e/0xa0
[ 207.927267] multi_cpu_stop+0xc4/0xf0
[ 207.927268] ? cpu_stop_queue_work+0xb0/0xb0
[ 207.927269] cpu_stopper_thread+0x96/0x120
[ 207.927270] smpboot_thread_fn+0x11a/0x1e0
[ 207.927272] kthread+0x10c/0x140
[ 207.927273] ? sort_range+0x30/0x30
[ 207.927274] ? kthread_parkme+0x40/0x40
[ 207.927276] ret_from_fork+0x2c/0x40
[ 207.927277] ---[ end trace 262c7e4b746d5a77 ]---
[ 207.927287] BUG: unable to handle kernel NULL pointer dereference at 0000000000000150
[ 207.927289] IP: pick_next_task_fair+0x397/0x550
[ 207.927289] PGD 0
[ 207.927290]
[ 207.927291] Oops: 0000 [#1] SMP
[ 207.927294] Dumping ftrace buffer:
[ 207.927296] (ftrace buffer empty)
[ 207.927296] Modules linked in: nls_utf8(E) isofs(E) ebtable_filter(E) ebtables(E) fuse(E) nf_log_ipv6(E) xt_pkttype(E) xt_physdev(E) br_netfilter(E) nf_log_ipv4(E) nf_log_common(E) xt_LOG(E) xt_limit(E) af_packet(E) bridge(E) stp(E) llc(E) iscsi_ibft(E) iscsi_boot_sysfs(E) ip6t_REJECT(E) xt_tcpudp(E) nf_conntrack_ipv6(E) nf_defrag_ipv6(E) ip6table_raw(E) ipt_REJECT(E) iptable_raw(E) xt_CT(E) iptable_filter(E) ip6table_mangle(E) nf_conntrack_netbios_ns(E) nf_conntrack_broadcast(E) nf_conntrack_ipv4(E) nf_defrag_ipv4(E) ip_tables(E) xt_conntrack(E) nf_conntrack(E) ip6table_filter(E) ip6_tables(E) x_tables(E) nls_iso8859_1(E) snd_hda_codec_hdmi(E) nls_cp437(E) intel_rapl(E) x86_pkg_temp_thermal(E) intel_powerclamp(E) coretemp(E) kvm_intel(E) snd_hda_codec_realtek(E) snd_hda_codec_generic(E) kvm(E) snd_hda_intel(E)
[ 207.927317] snd_hda_codec(E) irqbypass(E) crct10dif_pclmul(E) snd_hda_core(E) snd_hwdep(E) nfsd(E) crc32_pclmul(E) crc32c_intel(E) ghash_clmulni_intel(E) pcbc(E) snd_pcm(E) auth_rpcgss(E) aesni_intel(E) aes_x86_64(E) snd_timer(E) nfs_acl(E) joydev(E) crypto_simd(E) snd(E) lockd(E) grace(E) iTCO_wdt(E) iTCO_vendor_support(E) lpc_ich(E) mei_me(E) i2c_i801(E) mei(E) pcspkr(E) glue_helper(E) mfd_core(E) shpchp(E) intel_smartconnect(E) sunrpc(E) soundcore(E) tpm_infineon(E) fan(E) thermal(E) battery(E) cryptd(E) efivarfs(E) sr_mod(E) cdrom(E) hid_logitech_hidpp(E) hid_logitech_dj(E) uas(E) usb_storage(E) hid_generic(E) usbhid(E) nouveau(E) wmi(E) i2c_algo_bit(E) drm_kms_helper(E) ahci(E) syscopyarea(E) sysfillrect(E) sysimgblt(E) fb_sys_fops(E) libahci(E) xhci_pci(E) ehci_pci(E) xhci_hcd(E) ehci_hcd(E)
[ 207.927332] ttm(E) libata(E) r8169(E) mii(E) drm(E) usbcore(E) fjes(E) video(E) button(E) sd_mod(E) vfat(E) fat(E) ext4(E) crc16(E) jbd2(E) mbcache(E) dm_mod(E) loop(E) sg(E) scsi_mod(E) autofs4(E)
[ 207.927347] CPU: 6 PID: 45 Comm: migration/6 Tainted: G W E 4.10.0-tip-default #29
[ 207.927347] Hardware name: MEDION MS-7848/MS-7848, BIOS M7848W08.20C 09/23/2013
[ 207.927348] task: ffff88017df703c0 task.stack: ffffc90001a4c000
[ 207.927349] RIP: 0010:pick_next_task_fair+0x397/0x550
[ 207.927350] RSP: 0018:ffffc90001a4fdb8 EFLAGS: 00010006
[ 207.927350] RAX: 000000000b7eefb7 RBX: ffff88041ed98580 RCX: ffffffff8203e080
[ 207.927351] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000800
[ 207.927351] RBP: ffffc90001a4fe18 R08: 0000000000000001 R09: 0000000000000001
[ 207.927352] R10: 00000030696e879b R11: 00000000001b6d69 R12: 0000000000000000
[ 207.927352] R13: ffff88041ed985f0 R14: 0000000000000000 R15: 0000000000000000
[ 207.927353] FS: 0000000000000000(0000) GS:ffff88041ed80000(0000) knlGS:0000000000000000
[ 207.927354] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 207.927354] CR2: 0000000000000150 CR3: 0000000001c09000 CR4: 00000000001406e0
[ 207.927355] Call Trace:
[ 207.927358] __schedule+0x745/0x940
[ 207.927359] schedule+0x3d/0x90
[ 207.927360] __kthread_parkme+0x4e/0xa0
[ 207.927362] kthread_parkme+0x25/0x40
[ 207.927363] smpboot_thread_fn+0x96/0x1e0
[ 207.927364] kthread+0x10c/0x140
[ 207.927365] ? sort_range+0x30/0x30
[ 207.927366] ? kthread_parkme+0x40/0x40
[ 207.927366] ret_from_fork+0x2c/0x40
[ 207.927367] Code: 38 85 c9 74 c4 4c 89 e7 e8 d7 8e ff ff eb bd 4c 8b 75 b8 48 8d 48 80 48 89 4d c0 49 39 ce 0f 84 8b 00 00 00 49 83 ee 80 49 89 c4 <4d> 8b ac 24 50 01 00 00 49 8b be 50 01 00 00 49 39 fd 74 4d 41
[ 207.927377] RIP: pick_next_task_fair+0x397/0x550 RSP: ffffc90001a4fdb8
[ 207.927377] CR2: 0000000000000150