git checkout d8524ae9d6f492a9c6db9f4d89c5f9b8782fa2d5 Previous HEAD position was 1918664... Merge branch 'x86/intel-mid' HEAD is now at d8524ae... Merge branch 'drm-fixes' of git://people.freedesktop.org/~airlied/linux ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:d8524ae9d6f492a9c6db9f4d89c5f9b8782fa2d5:bisect-sound 2013-11-09-11:11:30 d8524ae9d6f492a9c6db9f4d89c5f9b8782fa2d5 compiling 943 real 1802 user 180 sys 210.32% cpu x86_64-randconfig-x4-1108 2013-11-09-11:34:14 detecting boot state 3.12.0-rc1-10002-gd8524ae... 2... 3........ 4. 5. 6. 8 10 12 15 17 19 23 25 27 SUCCESS bisect: good commit d8524ae9d6f492a9c6db9f4d89c5f9b8782fa2d5 git bisect start 1918664d0b2b6b2ad6b180e7c04a42872beb6691 d8524ae9d6f492a9c6db9f4d89c5f9b8782fa2d5 -- Previous HEAD position was d8524ae... Merge branch 'drm-fixes' of git://people.freedesktop.org/~airlied/linux HEAD is now at c1be5a5... Linux 3.9 Bisecting: 1785 revisions left to test after this (roughly 11 steps) [4d9218daae2b00379b2bb81eaf4fe5143f372db1] Merge tag 'perf-core-for-mingo' of git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux into perf/core git bisect run /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:4d9218daae2b00379b2bb81eaf4fe5143f372db1:bisect-sound 2013-11-09-11:50:23 4d9218daae2b00379b2bb81eaf4fe5143f372db1 reuse /kernel/x86_64-randconfig-x4-1108/4d9218daae2b00379b2bb81eaf4fe5143f372db1/vmlinuz-3.12.0-00312-g4d9218d 2013-11-09-11:50:24 detecting boot state ... 14 17 18 19 24 27 SUCCESS Bisecting: 892 revisions left to test after this (roughly 10 steps) [f05ef3f6f78e6b97f5664a8290f1ac0717b1a1b0] kvm tools: Switch to using an enum for disk image types running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:f05ef3f6f78e6b97f5664a8290f1ac0717b1a1b0:bisect-sound 2013-11-09-11:55:15 f05ef3f6f78e6b97f5664a8290f1ac0717b1a1b0 compiling 466 real 1130 user 108 sys 265.83% cpu x86_64-randconfig-x4-1108 2013-11-09-12:05:06 detecting boot state 3.0.0-rc5-00893-gf05ef3f... 3 14 22 26 27 SUCCESS Bisecting: 446 revisions left to test after this (roughly 9 steps) [3784592e0c0f224e399605cfa2d1570f79c0cc29] config: fix make kvmconfig running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:3784592e0c0f224e399605cfa2d1570f79c0cc29:bisect-sound 2013-11-09-12:09:18 3784592e0c0f224e399605cfa2d1570f79c0cc29 compiling 211 real 1414 user 144 sys 735.43% cpu x86_64-randconfig-x4-1108 2013-11-09-12:13:20 detecting boot state 3.7.0-rc1-01339-g3784592.. 1 7 16 23 26 27 SUCCESS Bisecting: 244 revisions left to test after this (roughly 8 steps) [9c028f0df729ba4958fe7b142d8521b66ee154d9] Merge branch 'sched/core' running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:9c028f0df729ba4958fe7b142d8521b66ee154d9:bisect-sound 2013-11-09-12:17:29 9c028f0df729ba4958fe7b142d8521b66ee154d9 compiling 531 real 1949 user 188 sys 402.34% cpu x86_64-randconfig-x4-1108 2013-11-09-12:27:25 detecting boot state 3.12.0-11925-g9c028f0.. 1.... 6. TEST FAILURE [ 12.682687] EDD information not available. [ 12.683565] Unregister pv shared memory for cpu 0 [ 12.852423] input: ImExPS/2 Generic Explorer Mouse as /devices/platform/i8042/serio1/input/input3 [ 242.583390] INFO: task swapper/0:1 blocked for more than 120 seconds. [ 242.584495] Not tainted 3.12.0-11925-g9c028f0 #402 [ 242.585371] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 242.586642] swapper/0 D ffff88000d88a040 0 1 0 0x00000000 [ 242.587874] ffff88000d88dc10 0000000000000046 ffffffff8261d260 ffff88000d88a040 [ 242.589188] ffff88000d88dfd8 0000000000012a00 0000000000012a00 ffffffff821ff4c0 [ 242.590509] ffff88000d88a040 ffff88000d88a040 ffff88000d88006f 0000000000000002 [ 242.591809] Call Trace: [ 242.592244] [] ? __lock_acquire+0x1e84/0x1f00 [ 242.593228] [] ? __lock_acquire+0x1e84/0x1f00 [ 242.594245] [] schedule+0x55/0x60 [ 242.595069] [] schedule_timeout+0x2b/0x200 [ 242.596012] [] ? mark_held_locks+0x12e/0x150 [ 242.597003] [] ? _raw_spin_unlock_irq+0x27/0x60 [ 242.598014] [] ? get_parent_ip+0xd/0x50 [ 242.598900] [] ? preempt_count_sub+0x33/0x50 [ 242.599879] [] wait_for_common+0x167/0x1b0 [ 242.600869] [] ? try_to_wake_up+0x330/0x330 [ 242.619936] [] ? __call_rcu.constprop.42+0x3d0/0x3d0 [ 242.621063] [] wait_for_completion+0x18/0x20 [ 242.622033] [] wait_rcu_gp+0x46/0x50 [ 242.622882] [] ? ftrace_raw_output_rcu_barrier+0x70/0x70 [ 242.624054] [] synchronize_rcu+0x2c/0x30 [ 242.624960] [] _cpu_down+0x2b2/0x340 [ 242.625834] [] ? mutex_lock_nested+0x3da/0x400 [ 242.626848] [] ? cpu_hotplug_driver_lock+0x12/0x20 [ 242.627912] [] cpu_down+0x31/0x50 [ 242.649858] [] _debug_hotplug_cpu+0x47/0xb0 [ 242.650853] [] ? pci_iommu_alloc+0x6c/0x6c [ 242.651784] [] debug_hotplug_cpu+0xd/0x11 [ 242.652698] [] do_one_initcall+0x7e/0x11b [ 242.653634] [] ? parse_args+0x1dd/0x300 [ 242.654521] [] kernel_init_freeable+0xfc/0x18d [ 242.655504] [] ? do_early_param+0x8a/0x8a [ 242.656419] [] ? rest_init+0xc0/0xc0 [ 242.657297] [] kernel_init+0x9/0x180 [ 242.671281] [] ret_from_fork+0x7c/0xb0 [ 242.672157] [] ? rest_init+0xc0/0xc0 [ 242.673028] 3 locks held by swapper/0/1: [ 242.673733] #0: (x86_cpu_hotplug_driver_mutex){+.+.+.}, at: [] cpu_hotplug_driver_lock+0x12/0x20 [ 242.675532] #1: (cpu_add_remove_lock){+.+.+.}, at: [] cpu_down+0x1d/0x50 [ 242.677035] #2: (cpu_hotplug.lock){+.+.+.}, at: [] cpu_hotplug_begin+0x26/0x60 [ 242.678590] sending NMI to all CPUs: [ 242.679197] NMI backtrace for cpu 1 [ 242.679773] CPU: 1 PID: 30 Comm: khungtaskd Not tainted 3.12.0-11925-g9c028f0 #402 [ 242.680357] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 242.680357] task: ffff8800073de7c0 ti: ffff8800073ea000 task.ti: ffff8800073ea000 [ 242.680357] RIP: 0010:[] [] flat_send_IPI_mask+0x8d/0xd0 [ 242.680357] RSP: 0000:ffff8800073ebdb8 EFLAGS: 00000046 [ 242.680357] RAX: 0000000003000000 RBX: 0000000000000002 RCX: 0000000000000007 [ 242.680357] RDX: 0000000000000c00 RSI: 0000000000000001 RDI: 0000000000000300 [ 242.680357] RBP: ffff8800073ebdd8 R08: 0000000000000000 R09: 0000000000000000 [ 242.680357] R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000246 [ 242.680357] R13: 0000000000000c00 R14: 0000000000000003 R15: 00000000000003ff [ 242.680357] FS: 0000000000000000(0000) GS:ffff88000df00000(0000) knlGS:0000000000000000 [ 242.680357] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 242.680357] CR2: 00000000ffffffff CR3: 00000000021fa000 CR4: 00000000000006a0 [ 242.680357] Stack: [ 242.680357] 0000000000002710 ffff88000d88a040 0000000000007fff ffff88000d88a040 [ 242.680357] ffff8800073ebde8 ffffffff8107385a ffff8800073ebe00 ffffffff81070abf [ 242.680357] 0000000000000078 ffff8800073ebe68 ffffffff8113e1b8 ffffffff8113df5b [ 242.680357] Call Trace: [ 242.680357] [] flat_send_IPI_all+0x1a/0x70 [ 242.680357] [] arch_trigger_all_cpu_backtrace+0x5f/0xb0 [ 242.680357] [] watchdog+0x2d8/0x3d0 [ 242.680357] [] ? watchdog+0x7b/0x3d0 [ 242.680357] [] ? hung_task_panic+0x20/0x20 [ 242.680357] [] kthread+0xd1/0xe0 [ 242.680357] [] ? __kthread_unpark+0x50/0x50 [ 242.680357] [] ret_from_fork+0x7c/0xb0 [ 242.680357] [] ? __kthread_unpark+0x50/0x50 [ 242.680357] Code: e6 10 75 f2 44 89 f0 c1 e0 18 89 04 25 10 b3 5b ff 89 da 44 09 ea 41 81 cd 00 04 00 00 83 fb 02 41 0f 44 d5 89 14 25 00 b3 5b ff <41> f7 c4 00 02 00 00 75 1a 4c 89 e7 57 9d 0f 1f 44 00 00 e8 4b /kernel/x86_64-randconfig-x4-1108/9c028f0df729ba4958fe7b142d8521b66ee154d9/dmesg-quantal-lkp-st01-8:20131109202717:x86_64-randconfig-x4-1108:3.12.0-11925-g9c028f0:402 /kernel/x86_64-randconfig-x4-1108/9c028f0df729ba4958fe7b142d8521b66ee154d9/dmesg-quantal-roam-1:20131109123151:x86_64-randconfig-x4-1108:3.12.0-11925-g9c028f0:402 4:2:4 all_good:bad:all_bad boots REPEAT COUNT: 50 # /cc/wfg/sound-bisect/.repeat Bisecting: 100 revisions left to test after this (roughly 7 steps) [04bb2f9475054298f0c67a89ca92cade42d3fe5e] sched/numa: Adjust scan rate in task_numa_placement running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:04bb2f9475054298f0c67a89ca92cade42d3fe5e:bisect-sound 2013-11-09-12:32:02 04bb2f9475054298f0c67a89ca92cade42d3fe5e compiling 258 real 1941 user 198 sys 828.53% cpu x86_64-randconfig-x4-1108 2013-11-09-12:39:46 detecting boot state 3.12.0-rc4-10777-g04bb2f9.. 1 10 13 18 22 26 34 43 49 50 SUCCESS Bisecting: 59 revisions left to test after this (roughly 6 steps) [ecf1f014325ba60f4df35edae1a357c67c5d4eb1] Merge branch 'core/rcu' into core/locking, to prepare the kernel/locking/ file move running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:ecf1f014325ba60f4df35edae1a357c67c5d4eb1:bisect-sound 2013-11-09-12:45:48 ecf1f014325ba60f4df35edae1a357c67c5d4eb1 compiling 280 real 1935 user 196 sys 759.29% cpu x86_64-randconfig-x4-1108 2013-11-09-12:53:46 detecting boot state 3.12.0-11460-gecf1f01. 1 4 29 46 50 SUCCESS Bisecting: 29 revisions left to test after this (roughly 5 steps) [01768b42dc97a67b4fb33a2535c49fc1969880df] locking: Move the mutex code to kernel/locking/ running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:01768b42dc97a67b4fb33a2535c49fc1969880df:bisect-sound 2013-11-09-12:56:47 01768b42dc97a67b4fb33a2535c49fc1969880df compiling 313 real 1947 user 196 sys 684.57% cpu x86_64-randconfig-x4-1108 2013-11-09-13:04:12 detecting boot state 3.12.0-11591-g01768b4. 4 8 16 20 21... TEST FAILURE BUG: kernel early hang without any printk output Command line: hung_task_panic=1 rcutree.rcu_cpu_stall_timeout=100 log_buf_len=8M ignore_loglevel debug sched_debug apic=debug dynamic_printk sysrq_always_enabled panic=10 prompt_ramdisk=0 console=ttyS0,115200 console=tty0 vga=normal root=/dev/ram0 rw link=/kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:01768b42dc97a67b4fb33a2535c49fc1969880df:bisect-sound/.vmlinuz-01768b42dc97a67b4fb33a2535c49fc1969880df-20131109130406-19-ant branch=tip/master BOOT_IMAGE=/kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/vmlinuz-3.12.0-11591-g01768b4 noapic nolapic nohz=off Early hang kernel: vmlinuz-3.12.0-11591-g01768b4 3.12.0-11591-g01768b4 #405 Elapsed time: 35 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-quantal-cairo-18:20131109130820:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-quantal-inn-37:20131109130821:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-yocto-inn-6:20131109130815:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-yocto-stoakley-5:20131109130822:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-yocto-inn-7:20131109130827:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-quantal-inn-35:20131109130829:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-quantal-jaketown-9:20131109130830:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-yocto-inn-12:20131109130827:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-yocto-inn-2:20131109130828:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 /kernel/x86_64-randconfig-x4-1108/01768b42dc97a67b4fb33a2535c49fc1969880df/dmesg-yocto-stoakley-3:20131109130832:x86_64-randconfig-x4-1108:3.12.0-11591-g01768b4:405 12:10:18 all_good:bad:all_bad boots Bisecting: 14 revisions left to test after this (roughly 4 steps) [c2d816443ef305aba8eaf0bf368f4d3d87494f06] sched/wait: Introduce prepare_to_wait_event() running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:c2d816443ef305aba8eaf0bf368f4d3d87494f06:bisect-sound 2013-11-09-13:08:47 c2d816443ef305aba8eaf0bf368f4d3d87494f06 compiling 325 real 1904 user 188 sys 642.62% cpu x86_64-randconfig-x4-1108 2013-11-09-13:17:05 detecting boot state 3.12.0-rc4-10855-gc2d8164... 2 3 4.... TEST FAILURE BUG: kernel early hang without any printk output Command line: hung_task_panic=1 rcutree.rcu_cpu_stall_timeout=100 log_buf_len=8M ignore_loglevel debug sched_debug apic=debug dynamic_printk sysrq_always_enabled panic=10 prompt_ramdisk=0 console=ttyS0,115200 console=tty0 vga=normal root=/dev/ram0 rw link=/kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:c2d816443ef305aba8eaf0bf368f4d3d87494f06:bisect-sound/.vmlinuz-c2d816443ef305aba8eaf0bf368f4d3d87494f06-20131109131702-31-ant branch=tip/master BOOT_IMAGE=/kernel/x86_64-randconfig-x4-1108/c2d816443ef305aba8eaf0bf368f4d3d87494f06/vmlinuz-3.12.0-rc4-10855-gc2d8164 noapic nolapic nohz=off Early hang kernel: vmlinuz-3.12.0-rc4-10855-gc2d8164 3.12.0-rc4-10855-gc2d8164 #406 Elapsed time: 35 /kernel/x86_64-randconfig-x4-1108/c2d816443ef305aba8eaf0bf368f4d3d87494f06/dmesg-yocto-roam-47:20131109132122:x86_64-randconfig-x4-1108:3.12.0-rc4-10855-gc2d8164:406 /kernel/x86_64-randconfig-x4-1108/c2d816443ef305aba8eaf0bf368f4d3d87494f06/dmesg-yocto-roam-43:20131109132137:x86_64-randconfig-x4-1108:3.12.0-rc4-10855-gc2d8164:406 /kernel/x86_64-randconfig-x4-1108/c2d816443ef305aba8eaf0bf368f4d3d87494f06/dmesg-quantal-athens-21:20131109132202:x86_64-randconfig-x4-1108:3.12.0-rc4-10855-gc2d8164:406 0:3:7 all_good:bad:all_bad boots Bisecting: 7 revisions left to test after this (roughly 3 steps) [ec0ad3d01f99d5e5b56a99a58f7003b99250dc65] Merge branch 'core/urgent' into sched/core running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:ec0ad3d01f99d5e5b56a99a58f7003b99250dc65:bisect-sound 2013-11-09-13:22:08 ec0ad3d01f99d5e5b56a99a58f7003b99250dc65 compiling 226 real 1932 user 189 sys 938.53% cpu x86_64-randconfig-x4-1108 2013-11-09-13:26:36 detecting boot state 3.12.0-rc4-10847-gec0ad3d.. 3 6 10 11.... 12 14 15 16 17. 19 20 21... 23... 24............ 25...... 27... 28 29 30..... 31.. 32... 33 34............ 38 40.. 42 47. 48 49 50 SUCCESS Bisecting: 3 revisions left to test after this (roughly 2 steps) [7c3f2ab7b844f1a859afbc3d41925e8a0faba5fa] sched/rt: Add missing rmb() running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:7c3f2ab7b844f1a859afbc3d41925e8a0faba5fa:bisect-sound 2013-11-09-14:11:10 7c3f2ab7b844f1a859afbc3d41925e8a0faba5fa compiling 234 real 1880 user 184 sys 880.28% cpu x86_64-randconfig-x4-1108 2013-11-09-14:15:17 detecting boot state 3.12.0-rc4-10851-g7c3f2ab... 1 2 3... 4 5 8 9. 10. 11 12 13. 14 17 18.... 19.......... 20............. 21. 22 24 25. 26. 27 29 30 32 35 40 44 46. 50 SUCCESS Bisecting: 1 revision left to test after this (roughly 1 step) [6acce3ef84520537f8a09a12c9ddbe814a584dd2] sched: Remove get_online_cpus() usage running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:6acce3ef84520537f8a09a12c9ddbe814a584dd2:bisect-sound 2013-11-09-14:50:19 6acce3ef84520537f8a09a12c9ddbe814a584dd2 compiling 37 real 76 user 49 sys 335.66% cpu x86_64-randconfig-x4-1108 2013-11-09-14:51:07 detecting boot state 3.12.0-rc4-10853-g6acce3e... 5.. 8 11 13 TEST FAILURE BUG: kernel early hang without any printk output Command line: hung_task_panic=1 rcutree.rcu_cpu_stall_timeout=100 log_buf_len=8M ignore_loglevel debug sched_debug apic=debug dynamic_printk sysrq_always_enabled panic=10 prompt_ramdisk=0 console=ttyS0,115200 console=tty0 vga=normal root=/dev/ram0 rw link=/kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:6acce3ef84520537f8a09a12c9ddbe814a584dd2:bisect-sound/.vmlinuz-6acce3ef84520537f8a09a12c9ddbe814a584dd2-20131109145104-32-ant branch=tip/master BOOT_IMAGE=/kernel/x86_64-randconfig-x4-1108/6acce3ef84520537f8a09a12c9ddbe814a584dd2/vmlinuz-3.12.0-rc4-10853-g6acce3e noapic nolapic nohz=off Early hang kernel: vmlinuz-3.12.0-rc4-10853-g6acce3e 3.12.0-rc4-10853-g6acce3e #409 Elapsed time: 35 /kernel/x86_64-randconfig-x4-1108/6acce3ef84520537f8a09a12c9ddbe814a584dd2/dmesg-yocto-xgwo-5:20131109225230:x86_64-randconfig-x4-1108:3.12.0-rc4-10853-g6acce3e:409 11:1:3 all_good:bad:all_bad boots REPEAT COUNT: 150 # /cc/wfg/sound-bisect/.repeat Bisecting: 0 revisions left to test after this (roughly 0 steps) [746023159c40c523b08a3bc3d213dac212385895] sched: Fix race in migrate_swap_stop() running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/sound/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:746023159c40c523b08a3bc3d213dac212385895:bisect-sound 2013-11-09-14:55:39 746023159c40c523b08a3bc3d213dac212385895 compiling 36 real 70 user 49 sys 326.84% cpu x86_64-randconfig-x4-1108 2013-11-09-14:56:49 detecting boot state 3.12.0-rc4-10852-g7460231.. 1 9 10 13 15 18 20 25 29 31 35 36 37 39. 43 44. 46 48 49 53 55. 56.. 58... 61 62.... 63 64 65. 66. 68 69 71. 72 75. 76. 78.. 81 83 84 89 91 93 95 99 103 107 110 114 119 128 134 146 150 SUCCESS 6acce3ef84520537f8a09a12c9ddbe814a584dd2 is the first bad commit commit 6acce3ef84520537f8a09a12c9ddbe814a584dd2 Author: Peter Zijlstra Date: Fri Oct 11 14:38:20 2013 +0200 sched: Remove get_online_cpus() usage Remove get_online_cpus() usage from the scheduler; there's 4 sites that use it: - sched_init_smp(); where its completely superfluous since we're in 'early' boot and there simply cannot be any hotplugging. - sched_getaffinity(); we already take a raw spinlock to protect the task cpus_allowed mask, this disables preemption and therefore also stabilizes cpu_online_mask as that's modified using stop_machine. However switch to active mask for symmetry with sched_setaffinity()/set_cpus_allowed_ptr(). We guarantee active mask stability by inserting sync_rcu/sched() into _cpu_down. - sched_setaffinity(); we don't appear to need get_online_cpus() either, there's two sites where hotplug appears relevant: * cpuset_cpus_allowed(); for the !cpuset case we use possible_mask, for the cpuset case we hold task_lock, which is a spinlock and thus for mainline disables preemption (might cause pain on RT). * set_cpus_allowed_ptr(); Holds all scheduler locks and thus has preemption properly disabled; also it already deals with hotplug races explicitly where it releases them. - migrate_swap(); we can make stop_two_cpus() do the heavy lifting for us with a little trickery. By adding a sync_sched/rcu() after the CPU_DOWN_PREPARE notifier we can provide preempt/rcu guarantees for cpu_active_mask. Use these to validate that both our cpus are active when queueing the stop work before we queue the stop_machine works for take_cpu_down(). Signed-off-by: Peter Zijlstra Cc: "Srivatsa S. Bhat" Cc: Paul McKenney Cc: Mel Gorman Cc: Rik van Riel Cc: Srikar Dronamraju Cc: Andrea Arcangeli Cc: Johannes Weiner Cc: Linus Torvalds Cc: Andrew Morton Cc: Steven Rostedt Cc: Oleg Nesterov Link: http://lkml.kernel.org/r/20131011123820.GV3081@twins.programming.kicks-ass.net Signed-off-by: Ingo Molnar :040000 040000 9f74473936bd4e621115b296bb747df5360a03b2 ad19ea9005a5ae9929f64a77d5e8883207788d34 M kernel bisect run success ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:746023159c40c523b08a3bc3d213dac212385895:bisect-sound 2013-11-09-15:33:57 746023159c40c523b08a3bc3d213dac212385895 reuse /kernel/x86_64-randconfig-x4-1108/746023159c40c523b08a3bc3d213dac212385895/vmlinuz-3.12.0-rc4-10852-g7460231 2013-11-09-15:33:58 detecting boot state .. 1 3. 4 5. 10 11 22 27 34 38 50 63 77 80 92 105 112 118 129 142 157 163 174 194 202 206. 212 213 216 220 228 229 237 240 247 251 253 262 269 271 287 301 312 331 345 355 366 380 392 397 410 430 437 445 449 450 SUCCESS ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:1918664d0b2b6b2ad6b180e7c04a42872beb6691:bisect-sound TEST FAILURE [ 4.941091] EDD information not available. [ 4.941871] Unregister pv shared memory for cpu 0 [ 5.325363] input: ImExPS/2 Generic Explorer Mouse as /devices/platform/i8042/serio1/input/input4 [ 241.303360] INFO: task swapper/0:1 blocked for more than 120 seconds. [ 241.304550] Not tainted 3.12.0-02098-g1918664 #98 [ 241.305492] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 241.306916] swapper/0 D ffff88000d8a2040 0 1 0 0x00000000 [ 241.308290] ffff88000d8a5c10 0000000000000046 ffffffff826aca60 ffff88000d8a2040 [ 241.309728] ffff88000d8a5fd8 0000000000012a00 0000000000012a00 ffff88000fa0c7c0 [ 241.311206] ffff88000d8a2040 ffff88000d8a2040 ffff88000d8a5b60 0000000000000002 [ 241.312642] Call Trace: [ 241.313111] [] ? __lock_acquire+0x1e74/0x1ef0 [ 241.314221] [] ? __lock_acquire+0x1e74/0x1ef0 [ 241.315307] [] schedule+0x55/0x60 [ 241.316210] [] schedule_timeout+0x2b/0x200 [ 241.317320] [] ? mark_held_locks+0x12e/0x150 [ 241.318390] [] ? _raw_spin_unlock_irq+0x27/0x60 [ 241.319508] [] ? get_parent_ip+0xd/0x50 [ 241.320517] [] ? preempt_count_sub+0x33/0x50 [ 241.321576] [] wait_for_common+0x167/0x1b0 [ 241.322612] [] ? try_to_wake_up+0x330/0x330 [ 241.323693] [] ? __call_rcu.constprop.42+0x3d0/0x3d0 [ 241.324893] [] wait_for_completion+0x18/0x20 [ 241.325965] [] wait_rcu_gp+0x46/0x50 [ 241.326937] [] ? ftrace_raw_output_rcu_barrier+0x70/0x70 [ 241.328228] [] synchronize_rcu+0x2c/0x30 [ 241.329234] [] _cpu_down+0x2b2/0x340 [ 241.330213] [] ? mutex_lock_nested+0x3da/0x400 [ 241.331319] [] ? cpu_hotplug_driver_lock+0x12/0x20 [ 241.332478] [] cpu_down+0x31/0x50 [ 241.333407] [] _debug_hotplug_cpu+0x47/0xb0 [ 241.334459] [] ? pci_iommu_alloc+0x6c/0x6c [ 241.335482] [] debug_hotplug_cpu+0xd/0x11 [ 241.336510] [] do_one_initcall+0x7e/0x11b [ 241.337606] [] ? parse_args+0x1dd/0x300 [ 241.338604] [] kernel_init_freeable+0xfc/0x18d [ 241.339700] [] ? do_early_param+0x8a/0x8a [ 241.340760] [] ? rest_init+0xc0/0xc0 [ 241.341695] [] kernel_init+0x9/0x180 [ 241.342658] [] ret_from_fork+0x7c/0xb0 [ 241.343657] [] ? rest_init+0xc0/0xc0 [ 241.346676] 3 locks held by swapper/0/1: [ 241.347410] #0: (x86_cpu_hotplug_driver_mutex){+.+.+.}, at: [] cpu_hotplug_driver_lock+0x12/0x20 [ 241.349432] #1: (cpu_add_remove_lock){+.+.+.}, at: [] cpu_down+0x1d/0x50 [ 241.351098] #2: (cpu_hotplug.lock){+.+.+.}, at: [] cpu_hotplug_begin+0x26/0x60 [ 241.352847] sending NMI to all CPUs: [ 241.353544] NMI backtrace for cpu 1 [ 241.354210] CPU: 1 PID: 29 Comm: khungtaskd Not tainted 3.12.0-02098-g1918664 #98 [ 241.355605] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 241.356699] task: ffff8800073fe780 ti: ffff88000f80a000 task.ti: ffff88000f80a000 [ 241.356846] RIP: 0010:[] [] flat_send_IPI_mask+0x8d/0xd0 [ 241.356846] RSP: 0000:ffff88000f80bdb8 EFLAGS: 00010046 [ 241.356846] RAX: 0000000003000000 RBX: 0000000000000002 RCX: 0000000000000007 [ 241.356846] RDX: 0000000000000c00 RSI: 0000000000000001 RDI: 0000000000000300 [ 241.356846] RBP: ffff88000f80bdd8 R08: 0000000000000000 R09: 0000000000000000 [ 241.356846] R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000246 [ 241.356846] R13: 0000000000000c00 R14: 0000000000000003 R15: 00000000000003ff [ 241.356846] FS: 0000000000000000(0000) GS:ffff88000dd00000(0000) knlGS:0000000000000000 [ 241.356846] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 241.356846] CR2: 00000000ffffffff CR3: 0000000002206000 CR4: 00000000000006a0 [ 241.356846] Stack: [ 241.356846] 0000000000002710 ffff88000d8a2040 0000000000007fff ffff88000d8a2040 [ 241.356846] ffff88000f80bde8 ffffffff810737da ffff88000f80be00 ffffffff81070a3f [ 241.356846] 0000000000000078 ffff88000f80be68 ffffffff8113e1a8 ffffffff8113df4b [ 241.356846] Call Trace: [ 241.356846] [] flat_send_IPI_all+0x1a/0x70 [ 241.356846] [] arch_trigger_all_cpu_backtrace+0x5f/0xb0 [ 241.356846] [] watchdog+0x2d8/0x3d0 [ 241.356846] [] ? watchdog+0x7b/0x3d0 [ 241.356846] [] ? hung_task_panic+0x20/0x20 [ 241.356846] [] kthread+0xd1/0xe0 [ 241.356846] [] ? _raw_spin_unlock_irq+0x27/0x60 [ 241.356846] [] ? __kthread_unpark+0x50/0x50 [ 241.356846] [] ret_from_fork+0x7c/0xb0 [ 241.356846] [] ? __kthread_unpark+0x50/0x50 [ 241.356846] Code: e6 10 75 f2 44 89 f0 c1 e0 18 89 04 25 10 b3 57 ff 89 da 44 09 ea 41 81 cd 00 04 00 00 83 fb 02 41 0f 44 d5 89 14 25 00 b3 57 ff <41> f7 c4 00 02 00 00 75 1a 4c 89 e7 57 9d 0f 1f 44 00 00 e8 8b /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-jaketown-20:20131108223032:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-stoakley-4:20131108223051:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-roam-3:20131109111057:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-waimea-1:20131109190803:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-stoakley-6:20131108223054:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-stoakley-2:20131108223514:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-quantal-lkp-tt02-15:20131109020106:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-quantal-athens-35:20131109110946:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-quantal-xps-5:20131109063342:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-stoakley-2:20131108223044:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-quantal-stoakley-6:20131108223332:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-lkp-st01-5:20131109062619:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 7:12:12 all_good:bad:all_bad boots [detached HEAD 1ae1820] Revert "sched: Remove get_online_cpus() usage" 3 files changed, 15 insertions(+), 48 deletions(-) ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:1ae1820bfe3f0964cf4bc3c5628055330e3cc538:bisect-sound 2013-11-09-16:04:32 1ae1820bfe3f0964cf4bc3c5628055330e3cc538 compiling 2013-11-09-16:08:45 detecting boot state 3.12.0-13518-g1ae1820... 3.. 6 7 16 24 36 56 78 98 118 140 163 181 198 212 234 251 270 277 282 297 312 317 323 326 330 332. 341 342 344 345.. 346 348 350 351 353. 355. 357 360 362 365 366 367 368 370 371 373. 374 376 377 378 381. 382 384 385 390 391 397 400 403 409 413 422 434 442 450 SUCCESS ========= upstream ========= Fetching linus ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:6c86ae2928f9e4cbf0d5844f5fcfd549e3450b8c:bisect-sound 2013-11-09-16:47:51 6c86ae2928f9e4cbf0d5844f5fcfd549e3450b8c compiling 2013-11-09-16:51:52 detecting boot state 3.12.0-13334-g6c86ae2.. 1 5 9 14. 29 33 36 37 42 49 56 66 72 87 96 107 108 113 125 133 139 150 160 167 174 178 183 192 196 207 209 223 229 236 245 258 265 275 281 290 298 309 318 322 325 332 335 341 346 349 350 352 353. 354. 362 370 377 394 402 413 427 434 437 443 450 SUCCESS ========= linux-next ========= Fetching next ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:bad570d77024cba808126d68a2072f6f8ce64c27:bisect-sound 2013-11-09-17:27:29 bad570d77024cba808126d68a2072f6f8ce64c27 compiling 2013-11-09-17:32:03 detecting boot state 3.12.0-next-20131108-22020-gbad570d.. 2 3 9 12 16 21 26 32 TEST FAILURE BUG: kernel early hang without any printk output Command line: hung_task_panic=1 rcutree.rcu_cpu_stall_timeout=100 log_buf_len=8M ignore_loglevel debug sched_debug apic=debug dynamic_printk sysrq_always_enabled panic=10 prompt_ramdisk=0 console=ttyS0,115200 console=tty0 vga=normal root=/dev/ram0 rw link=/kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:bad570d77024cba808126d68a2072f6f8ce64c27:bisect-sound/.vmlinuz-bad570d77024cba808126d68a2072f6f8ce64c27-20131109173159-221-ant branch=tip/master BOOT_IMAGE=/kernel/x86_64-randconfig-x4-1108/bad570d77024cba808126d68a2072f6f8ce64c27/vmlinuz-3.12.0-next-20131108-22020-gbad570d noapic nolapic nohz=off Early hang kernel: vmlinuz-3.12.0-next-20131108-22020-gbad570d 3.12.0-next-20131108-22020-gbad570d #413 Elapsed time: 35 /kernel/x86_64-randconfig-x4-1108/bad570d77024cba808126d68a2072f6f8ce64c27/dmesg-quantal-inn-27:20131109173655:x86_64-randconfig-x4-1108:3.12.0-next-20131108-22020-gbad570d:413 17:1:16 all_good:bad:all_bad boots ========= tip/master ========= Fetching tip ls -a /kernel-tests/run-queue/kvm/x86_64-randconfig-x4-1108/tip:master:1918664d0b2b6b2ad6b180e7c04a42872beb6691:bisect-sound TEST FAILURE [ 4.941091] EDD information not available. [ 4.941871] Unregister pv shared memory for cpu 0 [ 5.325363] input: ImExPS/2 Generic Explorer Mouse as /devices/platform/i8042/serio1/input/input4 [ 241.303360] INFO: task swapper/0:1 blocked for more than 120 seconds. [ 241.304550] Not tainted 3.12.0-02098-g1918664 #98 [ 241.305492] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 241.306916] swapper/0 D ffff88000d8a2040 0 1 0 0x00000000 [ 241.308290] ffff88000d8a5c10 0000000000000046 ffffffff826aca60 ffff88000d8a2040 [ 241.309728] ffff88000d8a5fd8 0000000000012a00 0000000000012a00 ffff88000fa0c7c0 [ 241.311206] ffff88000d8a2040 ffff88000d8a2040 ffff88000d8a5b60 0000000000000002 [ 241.312642] Call Trace: [ 241.313111] [] ? __lock_acquire+0x1e74/0x1ef0 [ 241.314221] [] ? __lock_acquire+0x1e74/0x1ef0 [ 241.315307] [] schedule+0x55/0x60 [ 241.316210] [] schedule_timeout+0x2b/0x200 [ 241.317320] [] ? mark_held_locks+0x12e/0x150 [ 241.318390] [] ? _raw_spin_unlock_irq+0x27/0x60 [ 241.319508] [] ? get_parent_ip+0xd/0x50 [ 241.320517] [] ? preempt_count_sub+0x33/0x50 [ 241.321576] [] wait_for_common+0x167/0x1b0 [ 241.322612] [] ? try_to_wake_up+0x330/0x330 [ 241.323693] [] ? __call_rcu.constprop.42+0x3d0/0x3d0 [ 241.324893] [] wait_for_completion+0x18/0x20 [ 241.325965] [] wait_rcu_gp+0x46/0x50 [ 241.326937] [] ? ftrace_raw_output_rcu_barrier+0x70/0x70 [ 241.328228] [] synchronize_rcu+0x2c/0x30 [ 241.329234] [] _cpu_down+0x2b2/0x340 [ 241.330213] [] ? mutex_lock_nested+0x3da/0x400 [ 241.331319] [] ? cpu_hotplug_driver_lock+0x12/0x20 [ 241.332478] [] cpu_down+0x31/0x50 [ 241.333407] [] _debug_hotplug_cpu+0x47/0xb0 [ 241.334459] [] ? pci_iommu_alloc+0x6c/0x6c [ 241.335482] [] debug_hotplug_cpu+0xd/0x11 [ 241.336510] [] do_one_initcall+0x7e/0x11b [ 241.337606] [] ? parse_args+0x1dd/0x300 [ 241.338604] [] kernel_init_freeable+0xfc/0x18d [ 241.339700] [] ? do_early_param+0x8a/0x8a [ 241.340760] [] ? rest_init+0xc0/0xc0 [ 241.341695] [] kernel_init+0x9/0x180 [ 241.342658] [] ret_from_fork+0x7c/0xb0 [ 241.343657] [] ? rest_init+0xc0/0xc0 [ 241.346676] 3 locks held by swapper/0/1: [ 241.347410] #0: (x86_cpu_hotplug_driver_mutex){+.+.+.}, at: [] cpu_hotplug_driver_lock+0x12/0x20 [ 241.349432] #1: (cpu_add_remove_lock){+.+.+.}, at: [] cpu_down+0x1d/0x50 [ 241.351098] #2: (cpu_hotplug.lock){+.+.+.}, at: [] cpu_hotplug_begin+0x26/0x60 [ 241.352847] sending NMI to all CPUs: [ 241.353544] NMI backtrace for cpu 1 [ 241.354210] CPU: 1 PID: 29 Comm: khungtaskd Not tainted 3.12.0-02098-g1918664 #98 [ 241.355605] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 241.356699] task: ffff8800073fe780 ti: ffff88000f80a000 task.ti: ffff88000f80a000 [ 241.356846] RIP: 0010:[] [] flat_send_IPI_mask+0x8d/0xd0 [ 241.356846] RSP: 0000:ffff88000f80bdb8 EFLAGS: 00010046 [ 241.356846] RAX: 0000000003000000 RBX: 0000000000000002 RCX: 0000000000000007 [ 241.356846] RDX: 0000000000000c00 RSI: 0000000000000001 RDI: 0000000000000300 [ 241.356846] RBP: ffff88000f80bdd8 R08: 0000000000000000 R09: 0000000000000000 [ 241.356846] R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000246 [ 241.356846] R13: 0000000000000c00 R14: 0000000000000003 R15: 00000000000003ff [ 241.356846] FS: 0000000000000000(0000) GS:ffff88000dd00000(0000) knlGS:0000000000000000 [ 241.356846] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 241.356846] CR2: 00000000ffffffff CR3: 0000000002206000 CR4: 00000000000006a0 [ 241.356846] Stack: [ 241.356846] 0000000000002710 ffff88000d8a2040 0000000000007fff ffff88000d8a2040 [ 241.356846] ffff88000f80bde8 ffffffff810737da ffff88000f80be00 ffffffff81070a3f [ 241.356846] 0000000000000078 ffff88000f80be68 ffffffff8113e1a8 ffffffff8113df4b [ 241.356846] Call Trace: [ 241.356846] [] flat_send_IPI_all+0x1a/0x70 [ 241.356846] [] arch_trigger_all_cpu_backtrace+0x5f/0xb0 [ 241.356846] [] watchdog+0x2d8/0x3d0 [ 241.356846] [] ? watchdog+0x7b/0x3d0 [ 241.356846] [] ? hung_task_panic+0x20/0x20 [ 241.356846] [] kthread+0xd1/0xe0 [ 241.356846] [] ? _raw_spin_unlock_irq+0x27/0x60 [ 241.356846] [] ? __kthread_unpark+0x50/0x50 [ 241.356846] [] ret_from_fork+0x7c/0xb0 [ 241.356846] [] ? __kthread_unpark+0x50/0x50 [ 241.356846] Code: e6 10 75 f2 44 89 f0 c1 e0 18 89 04 25 10 b3 57 ff 89 da 44 09 ea 41 81 cd 00 04 00 00 83 fb 02 41 0f 44 d5 89 14 25 00 b3 57 ff <41> f7 c4 00 02 00 00 75 1a 4c 89 e7 57 9d 0f 1f 44 00 00 e8 8b /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-jaketown-20:20131108223032:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-stoakley-4:20131108223051:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-roam-3:20131109111057:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-waimea-1:20131109190803:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-stoakley-6:20131108223054:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-stoakley-2:20131108223514:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-quantal-lkp-tt02-15:20131109020106:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-quantal-athens-35:20131109110946:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-quantal-xps-5:20131109063342:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-stoakley-2:20131108223044:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-quantal-stoakley-6:20131108223332:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 /kernel/x86_64-randconfig-x4-1108/1918664d0b2b6b2ad6b180e7c04a42872beb6691/dmesg-yocto-lkp-st01-5:20131109062619:x86_64-randconfig-x4-1108:3.12.0-02098-g1918664:98 7:12:12 all_good:bad:all_bad boots