git checkout 874db4d8002f89e1494be9d5c32f298cec621251 Previous HEAD position was 8b5ede6... powerpc/irq: Don't switch to irq stack from softirq stack HEAD is now at 874db4d... Merge branch 'fixes' of git://git.linaro.org/people/rmk/linux-arm ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:874db4d8002f89e1494be9d5c32f298cec621251:bisect-mm 2013-10-08-14:18:10 874db4d8002f89e1494be9d5c32f298cec621251 compiling 1562 real 4991 user 425 sys 346.78% cpu x86_64-allyesdebian 2013-10-08-14:45:53 detecting boot state 3.12.0-rc2-00113-g874db4d. TEST FAILURE [ 1.253674] pci 0000:00:02.0: Boot video device [ 1.254879] PCI: CLS 0 bytes, default 64 [ 1.418120] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 1.419493] swapper/0 cpuset=/ mems_allowed=0 [ 1.420280] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.12.0-rc2-00113-g874db4d #416 [ 1.421622] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 1.422577] 0000000000000002 ffff88001ef7bb28 ffffffff82c61952 ffff88001ef7c040 [ 1.424235] ffff88001ef7bba8 ffffffff82c57b22 ffffffff83c54680 ffff88001ef7bb60 [ 1.425879] ffffffff810f3e6e 000000000000143a 0000000000000246 ffff88001ef7bb98 [ 1.427514] Call Trace: [ 1.428059] [] dump_stack+0x54/0x74 [ 1.428946] [] dump_header.isra.10+0x7a/0x1ba [ 1.429936] [] ? lock_release_holdtime.part.27+0x4c/0x50 [ 1.431040] [] ? lock_release+0x189/0x1d1 [ 1.431985] [] out_of_memory+0x39e/0x3ee [ 1.432928] [] __alloc_pages_nodemask+0x668/0x7de [ 1.433959] [] kmem_getpages+0x75/0x16c [ 1.434867] [] fallback_alloc+0x12c/0x1ea [ 1.435798] [] ? trace_hardirqs_off+0xd/0xf [ 1.436765] [] ____cache_alloc_node+0x14a/0x159 [ 1.437754] [] ? dma_debug_init+0x1ef/0x29a [ 1.438702] [] kmem_cache_alloc_trace+0x83/0x11a [ 1.439704] [] dma_debug_init+0x1ef/0x29a [ 1.440667] [] pci_iommu_init+0x16/0x52 [ 1.441589] [] ? iommu_setup+0x27d/0x27d [ 1.442532] [] do_one_initcall+0x93/0x137 [ 1.443488] [] ? parameqn+0x28/0x3b [ 1.444371] [] ? parse_args+0x189/0x247 [ 1.445275] [] kernel_init_freeable+0x15e/0x1df [ 1.446273] [] ? do_early_param+0x88/0x88 [ 1.447186] [] ? rest_init+0xdb/0xdb [ 1.448085] [] kernel_init+0xe/0xdb [ 1.448968] [] ret_from_fork+0x7c/0xb0 [ 1.449882] [] ? rest_init+0xdb/0xdb [ 1.450784] Mem-Info: [ 1.451291] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-yocto-waimea-13:20131008224301:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-yocto-waimea-9:20131008224303:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-quantal-lkp-st01-10:20131008144611:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-quantal-xps-7:20131008224237:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-nfsroot-waimea-12:20131008224318:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-nfsroot-waimea-9:20131008224318:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-quantal-lkp-st01-9:20131008144620:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-quantal-lkp-tt02-13:20131008051712:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 /kernel/x86_64-allyesdebian/874db4d8002f89e1494be9d5c32f298cec621251/dmesg-yocto-lkp-st01-5:20131008144617:x86_64-allyesdebian:3.12.0-rc2-00113-g874db4d:416 0:9:5 all_good:bad:all_bad boots bisect: bad commit 874db4d8002f89e1494be9d5c32f298cec621251 git checkout v3.11 Previous HEAD position was 874db4d... Merge branch 'fixes' of git://git.linaro.org/people/rmk/linux-arm HEAD is now at 6e46645... Linux 3.11 ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:6e4664525b1db28f8c4e1130957f70a94c19213e:bisect-mm 2013-10-08-14:46:29 6e4664525b1db28f8c4e1130957f70a94c19213e compiling 1139 real 4883 user 424 sys 465.98% cpu x86_64-allyesdebian 2013-10-08-15:05:51 detecting boot state 3.11.0.................... 11 20 SUCCESS bisect: good commit v3.11 git bisect start 874db4d8002f89e1494be9d5c32f298cec621251 v3.11 -- Previous HEAD position was 6e46645... Linux 3.11 HEAD is now at 3c0eee3... Linux 2.6.37 Bisecting: 5600 revisions left to test after this (roughly 12 steps) [57d730924d5cc2c3e280af16a9306587c3a511db] Merge branch 'timers-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip git bisect run /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:57d730924d5cc2c3e280af16a9306587c3a511db:bisect-mm 2013-10-08-15:17:26 57d730924d5cc2c3e280af16a9306587c3a511db compiling 1056 real 4915 user 438 sys 506.99% cpu x86_64-allyesdebian 2013-10-08-15:35:53 detecting boot state 3.11.0-04694-g57d7309.................... 17 18 20 SUCCESS Bisecting: 2791 revisions left to test after this (roughly 12 steps) [c4c17252283a13c0d63a8d9df828da109c116411] Merge tag 'ntb-3.12' of git://github.com/jonmason/ntb running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:c4c17252283a13c0d63a8d9df828da109c116411:bisect-mm 2013-10-08-15:47:26 c4c17252283a13c0d63a8d9df828da109c116411 compiling 1301 real 4940 user 443 sys 413.53% cpu x86_64-allyesdebian 2013-10-08-16:09:44 detecting boot state 3.11.0-07503-gc4c1725.................... 13 20 SUCCESS Bisecting: 1395 revisions left to test after this (roughly 11 steps) [ffbbb96dd7570b9aafd426cd77a7ee03d224cabf] firmware/dmi_scan: constify strings running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:ffbbb96dd7570b9aafd426cd77a7ee03d224cabf:bisect-mm 2013-10-08-16:20:49 ffbbb96dd7570b9aafd426cd77a7ee03d224cabf compiling 1085 real 4929 user 432 sys 493.84% cpu x86_64-allyesdebian 2013-10-08-16:41:07 detecting boot state 3.11.0-09275-gffbbb96. TEST FAILURE [ 1.462347] pci 0000:00:02.0: Boot video device [ 1.463272] PCI: CLS 0 bytes, default 64 [ 1.940892] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 1.942390] swapper/0 cpuset=/ mems_allowed=0 [ 1.943201] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.11.0-09275-gffbbb96 #420 [ 1.944604] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 1.945590] 0000000000000002 ffff88001ef79b18 ffffffff82c55acf ffff88001ef74040 [ 1.947281] ffff88001ef79b98 ffffffff82c4de56 ffffffff83c54580 ffff88001ef79b50 [ 1.948993] ffffffff810ee077 0000000000007ab1 0000000000000246 ffff88001ef79b88 [ 1.950684] Call Trace: [ 1.951253] [] dump_stack+0x54/0x74 [ 1.952177] [] dump_header.isra.10+0x7a/0x1ba [ 1.953205] [] ? lock_release_holdtime.part.27+0x4c/0x50 [ 1.954373] [] ? lock_release+0x189/0x1d1 [ 1.955352] [] out_of_memory+0x39e/0x3ee [ 1.956336] [] __alloc_pages_nodemask+0x668/0x7de [ 1.957420] [] kmem_getpages+0x75/0x16c [ 1.958367] [] fallback_alloc+0x12c/0x1ea [ 1.959340] [] ? trace_hardirqs_off+0xd/0xf [ 1.960368] [] ____cache_alloc_node+0x14a/0x159 [ 1.961410] [] __kmalloc+0x95/0x12d [ 1.962317] [] ? kzalloc.constprop.16+0xe/0x10 [ 1.963337] [] kzalloc.constprop.16+0xe/0x10 [ 1.964375] [] dma_debug_init+0x1e3/0x28e [ 1.965352] [] pci_iommu_init+0x16/0x52 [ 1.966304] [] ? iommu_setup+0x27d/0x27d [ 1.967268] [] do_one_initcall+0x93/0x137 [ 1.968254] [] ? parameq+0x4/0x24 [ 1.969137] [] ? parse_args+0x189/0x247 [ 1.970093] [] kernel_init_freeable+0x15e/0x1df [ 1.971131] [] ? do_early_param+0x88/0x88 [ 1.972119] [] ? rest_init+0xdb/0xdb [ 1.973033] [] kernel_init+0xe/0xdb [ 1.973930] [] ret_from_fork+0x7c/0xb0 [ 1.974893] [] ? rest_init+0xdb/0xdb [ 1.975800] Mem-Info: [ 1.976375] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-nfsroot-athens-23:20131008164119:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-nfsroot-cairo-19:20131008164116:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-nfsroot-cairo-7:20131008164116:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-nfsroot-roam-14:20131008164115:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-quantal-athens-44:20131008164116:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-quantal-cairo-16:20131008164115:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-quantal-inn-14:20131008164116:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-quantal-inn-36:20131008164118:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-athens-35:20131008164115:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-athens-37:20131008164115:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-cairo-14:20131008164116:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-cairo-40:20131008164115:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-inn-35:20131008164116:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-roam-21:20131008164115:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-waimea-15:20131009003814:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-waimea-6:20131009003813:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-waimea-7:20131009003813:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-quantal-lkp-st01-6:20131008164125:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-lkp-tt02-16:20131008071221:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 /kernel/x86_64-allyesdebian/ffbbb96dd7570b9aafd426cd77a7ee03d224cabf/dmesg-yocto-stoakley-2:20131008164125:x86_64-allyesdebian:3.11.0-09275-gffbbb96:420 0:20:20 all_good:bad:all_bad boots REPEAT COUNT: 20 # /cc/wfg/mm-bisect/.repeat Bisecting: 707 revisions left to test after this (roughly 10 steps) [a35c6322e52c550b61a04a44df27d22394ee0a2c] Merge tag 'drivers-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/arm/arm-soc running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:a35c6322e52c550b61a04a44df27d22394ee0a2c:bisect-mm 2013-10-08-16:41:40 a35c6322e52c550b61a04a44df27d22394ee0a2c compiling 1103 real 4867 user 417 sys 478.98% cpu x86_64-allyesdebian 2013-10-08-17:00:42 detecting boot state 3.11.0-08567-ga35c632.................... 20 SUCCESS Bisecting: 347 revisions left to test after this (roughly 9 steps) [ec5b103ecfde929004b691f29183255aeeadecd5] Merge branch 'for-linus' of git://git.infradead.org/users/vkoul/slave-dma running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:ec5b103ecfde929004b691f29183255aeeadecd5:bisect-mm 2013-10-08-17:11:14 ec5b103ecfde929004b691f29183255aeeadecd5 compiling 698 real 4972 user 438 sys 774.76% cpu x86_64-allyesdebian 2013-10-08-17:23:10 detecting boot state 3.11.0-08927-gec5b103.................... 4 20 SUCCESS Bisecting: 173 revisions left to test after this (roughly 8 steps) [e1403b8edf669ff49bbdf602cc97fefa2760cb15] include/linux/sched.h: don't use task->pid/tgid in same_thread_group/has_group_leader_pid running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:e1403b8edf669ff49bbdf602cc97fefa2760cb15:bisect-mm 2013-10-08-17:34:12 e1403b8edf669ff49bbdf602cc97fefa2760cb15 compiling 746 real 4927 user 429 sys 717.22% cpu x86_64-allyesdebian 2013-10-08-17:46:58 detecting boot state 3.11.0-09101-ge1403b8.................... 2 16 20 SUCCESS Bisecting: 86 revisions left to test after this (roughly 7 steps) [7225522bb429a2f7dae6667e533e2d735b4882d0] mm: munlock: batch non-THP page isolation and munlock+putback using pagevec running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:7225522bb429a2f7dae6667e533e2d735b4882d0:bisect-mm 2013-10-08-17:58:28 7225522bb429a2f7dae6667e533e2d735b4882d0 compiling 731 real 4927 user 423 sys 732.00% cpu x86_64-allyesdebian 2013-10-08-18:10:57 detecting boot state 3.11.0-09188-g7225522.................... 7 17 20 SUCCESS Bisecting: 43 revisions left to test after this (roughly 6 steps) [fa688207c9db48b64ab6538abc3fcdf26110b9ec] smp: quit unconditionally enabling irq in on_each_cpu_mask and on_each_cpu_cond running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:fa688207c9db48b64ab6538abc3fcdf26110b9ec:bisect-mm 2013-10-08-18:22:28 fa688207c9db48b64ab6538abc3fcdf26110b9ec compiling 724 real 4960 user 434 sys 745.03% cpu x86_64-allyesdebian 2013-10-08-18:34:47 detecting boot state 3.11.0-09231-gfa68820. TEST FAILURE [ 1.802025] pci 0000:00:02.0: Boot video device [ 1.803531] PCI: CLS 0 bytes, default 64 [ 2.110982] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 2.113492] swapper/0 cpuset=/ mems_allowed=0 [ 2.114834] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.11.0-09231-gfa68820 #425 [ 2.117064] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 2.118690] 0000000000000002 ffff88001ef79b18 ffffffff82c55a8f ffff88001ef74040 [ 2.121479] ffff88001ef79b98 ffffffff82c4de16 ffffffff83c54580 ffff88001ef79b50 [ 2.124278] ffffffff810ee077 0000000000001be1 0000000000000246 ffff88001ef79b88 [ 2.126943] Call Trace: [ 2.127902] [] dump_stack+0x54/0x74 [ 2.129450] [] dump_header.isra.10+0x7a/0x1ba [ 2.131120] [] ? lock_release_holdtime.part.27+0x4c/0x50 [ 2.133012] [] ? lock_release+0x189/0x1d1 [ 2.134586] [] out_of_memory+0x39e/0x3ee [ 2.136223] [] __alloc_pages_nodemask+0x668/0x7de [ 2.138081] [] kmem_getpages+0x75/0x16c [ 2.139585] [] fallback_alloc+0x12c/0x1ea [ 2.141231] [] ? trace_hardirqs_off+0xd/0xf [ 2.142794] [] ____cache_alloc_node+0x14a/0x159 [ 2.144506] [] __kmalloc+0x95/0x12d [ 2.145987] [] ? kzalloc.constprop.16+0xe/0x10 [ 2.147633] [] kzalloc.constprop.16+0xe/0x10 [ 2.149374] [] dma_debug_init+0x1e3/0x28e [ 2.150963] [] pci_iommu_init+0x16/0x52 [ 2.152550] [] ? iommu_setup+0x27d/0x27d [ 2.154078] [] do_one_initcall+0x93/0x137 [ 2.155654] [] ? parameq+0x4/0x24 [ 2.157119] [] ? parse_args+0x189/0x247 [ 2.158698] [] kernel_init_freeable+0x15e/0x1df [ 2.160437] [] ? do_early_param+0x88/0x88 [ 2.161987] [] ? rest_init+0xdb/0xdb [ 2.163391] [] kernel_init+0xe/0xdb [ 2.164875] [] ret_from_fork+0x7c/0xb0 [ 2.166358] [] ? rest_init+0xdb/0xdb [ 2.167918] Mem-Info: [ 2.168847] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-nfsroot-inn-9:20131008183503:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-nfsroot-xian-30:20131008183501:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-nfsroot-cairo-19:20131008183512:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-nfsroot-waimea-2:20131009023204:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-nfsroot-xian-6:20131008183512:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-nfsroot-cairo-28:20131008183513:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-nfsroot-cairo-31:20131008183513:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-yocto-snb-4:20131008183513:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-nfsroot-waimea-10:20131009023211:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 /kernel/x86_64-allyesdebian/fa688207c9db48b64ab6538abc3fcdf26110b9ec/dmesg-quantal-bay-1:20131008183516:x86_64-allyesdebian:3.11.0-09231-gfa68820:425 0:10:7 all_good:bad:all_bad boots Bisecting: 21 revisions left to test after this (roughly 5 steps) [b194b8cdb83daafd2405fb902193b8e904107614] mm/hwpoison: add '#' to madvise_hwpoison running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:b194b8cdb83daafd2405fb902193b8e904107614:bisect-mm 2013-10-08-18:35:20 b194b8cdb83daafd2405fb902193b8e904107614 compiling 731 real 4954 user 423 sys 735.15% cpu x86_64-allyesdebian 2013-10-08-18:47:45 detecting boot state 3.11.0-09209-gb194b8c... TEST FAILURE [ 1.913424] pci 0000:00:02.0: Boot video device [ 1.914589] PCI: CLS 0 bytes, default 64 [ 2.190775] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 2.192853] swapper/0 cpuset=/ mems_allowed=0 [ 2.193999] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.11.0-09209-gb194b8c #426 [ 2.195879] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 2.197434] 0000000000000002 ffff88001ef79b18 ffffffff82c559af ffff88001ef74040 [ 2.199951] ffff88001ef79b98 ffffffff82c4dd36 ffffffff83c54580 ffff88001ef79b50 [ 2.202512] ffffffff810ee06e 00000000000024c1 0000000000000246 ffff88001ef79b88 [ 2.205120] Call Trace: [ 2.205974] [] dump_stack+0x54/0x74 [ 2.207422] [] dump_header.isra.10+0x7a/0x1ba [ 2.208895] [] ? lock_release_holdtime.part.27+0x4c/0x50 [ 2.210459] [] ? lock_release+0x189/0x1d1 [ 2.211822] [] out_of_memory+0x39e/0x3ee [ 2.213218] [] __alloc_pages_nodemask+0x668/0x7de [ 2.214681] [] kmem_getpages+0x75/0x16c [ 2.215986] [] fallback_alloc+0x12c/0x1ea [ 2.217526] [] ? trace_hardirqs_off+0xd/0xf [ 2.219011] [] ____cache_alloc_node+0x14a/0x159 [ 2.220606] [] __kmalloc+0x95/0x12d [ 2.221959] [] ? kzalloc.constprop.16+0xe/0x10 [ 2.223500] [] kzalloc.constprop.16+0xe/0x10 [ 2.225048] [] dma_debug_init+0x1e3/0x28e [ 2.226459] [] pci_iommu_init+0x16/0x52 [ 2.227767] [] ? iommu_setup+0x27d/0x27d [ 2.229170] [] do_one_initcall+0x93/0x137 [ 2.230530] [] ? parameq+0xd/0x24 [ 2.231758] [] ? parse_args+0x189/0x247 [ 2.233095] [] kernel_init_freeable+0x15e/0x1df [ 2.234512] [] ? do_early_param+0x88/0x88 [ 2.235874] [] ? rest_init+0xdb/0xdb [ 2.237315] [] kernel_init+0xe/0xdb [ 2.238669] [] ret_from_fork+0x7c/0xb0 [ 2.240105] [] ? rest_init+0xdb/0xdb [ 2.241472] Mem-Info: [ 2.242286] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-nfsroot-inn-19:20131008184801:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-nfsroot-inn-20:20131008184800:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-nfsroot-lkp-tt02-10:20131008091905:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-nfsroot-roam-30:20131008184805:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-nfsroot-waimea-5:20131009024453:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-quantal-inn-15:20131008184802:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-quantal-inn-24:20131008184800:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-quantal-inn-28:20131008184754:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-quantal-inn-29:20131008184754:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-quantal-inn-47:20131008184755:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-quantal-jaketown-6:20131008184809:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-quantal-stoakley-4:20131008184803:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-yocto-inn-23:20131008184800:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-yocto-inn-3:20131008184801:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-yocto-jaketown-23:20131008184810:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-yocto-jaketown-26:20131008184806:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-yocto-jaketown-28:20131008184807:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-yocto-lkp-tt02-4:20131008091903:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-yocto-lkp-tt02-5:20131008091910:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 /kernel/x86_64-allyesdebian/b194b8cdb83daafd2405fb902193b8e904107614/dmesg-yocto-lkp-tt02-8:20131008091904:x86_64-allyesdebian:3.11.0-09209-gb194b8c:426 0:20:17 all_good:bad:all_bad boots Bisecting: 10 revisions left to test after this (roughly 3 steps) [4c3bffc272755c98728c2b58b1a8148cf9e9fd1f] mm/backing-dev.c: check user buffer length before copying data to the related user buffer running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:4c3bffc272755c98728c2b58b1a8148cf9e9fd1f:bisect-mm 2013-10-08-18:49:18 4c3bffc272755c98728c2b58b1a8148cf9e9fd1f compiling 754 real 4945 user 435 sys 713.63% cpu x86_64-allyesdebian 2013-10-08-19:02:18 detecting boot state 3.11.0-09198-g4c3bffc. TEST FAILURE [ 1.745345] pci 0000:00:02.0: Boot video device [ 1.747278] PCI: CLS 0 bytes, default 64 [ 2.088981] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 2.091346] swapper/0 cpuset=/ mems_allowed=0 [ 2.092627] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.11.0-09198-g4c3bffc #427 [ 2.094780] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 2.096326] 0000000000000002 ffff88001ef79b18 ffffffff82c5582d ffff88001ef74040 [ 2.099009] ffff88001ef79b98 ffffffff82c4dcb6 ffffffff83c54580 ffff88001ef79b50 [ 2.101762] ffffffff810ee06e 0000000000002821 0000000000000246 ffff88001ef79b88 [ 2.104427] Call Trace: [ 2.105362] [] dump_stack+0x54/0x74 [ 2.106740] [] dump_header.isra.10+0x7a/0x1ba [ 2.108357] [] ? lock_release_holdtime.part.27+0x4c/0x50 [ 2.110197] [] ? lock_release+0x189/0x1d1 [ 2.111719] [] out_of_memory+0x39e/0x3ee [ 2.113302] [] __alloc_pages_nodemask+0x668/0x7de [ 2.114990] [] kmem_getpages+0x75/0x16c [ 2.116514] [] fallback_alloc+0x12c/0x1ea [ 2.118043] [] ? trace_hardirqs_off+0xd/0xf [ 2.119452] [] ____cache_alloc_node+0x14a/0x159 [ 2.121168] [] __kmalloc+0x95/0x12d [ 2.122539] [] ? kzalloc.constprop.16+0xe/0x10 [ 2.124204] [] kzalloc.constprop.16+0xe/0x10 [ 2.125755] [] dma_debug_init+0x1e3/0x28e [ 2.127259] [] pci_iommu_init+0x16/0x52 [ 2.128815] [] ? iommu_setup+0x27d/0x27d [ 2.130344] [] do_one_initcall+0x93/0x137 [ 2.131859] [] ? parameq+0xd/0x24 [ 2.133245] [] ? parse_args+0x189/0x247 [ 2.134714] [] kernel_init_freeable+0x15e/0x1df [ 2.136333] [] ? do_early_param+0x88/0x88 [ 2.137770] [] ? rest_init+0xdb/0xdb [ 2.139166] [] kernel_init+0xe/0xdb [ 2.140718] [] ret_from_fork+0x7c/0xb0 [ 2.142172] [] ? rest_init+0xdb/0xdb [ 2.143606] Mem-Info: [ 2.144504] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-nfsroot-waimea-1:20131009025924:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-nfsroot-waimea-4:20131009025921:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-quantal-xps-2:20131009025850:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-nfsroot-inn-1:20131008190231:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-nfsroot-waimea-4:20131009025931:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-nfsroot-waimea-9:20131009025929:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-quantal-cairo-40:20131008190236:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-quantal-inn-25:20131008190233:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-quantal-lkp-st01-3:20131008190234:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-quantal-lkp-tt02-21:20131008093336:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-yocto-bens-1:20131008190236:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-yocto-cairo-48:20131008190236:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-yocto-lkp-tt02-21:20131008093336:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-nfsroot-waimea-1:20131009025933:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-yocto-stoakley-2:20131008190240:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-yocto-lkp-st01-9:20131008190241:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 /kernel/x86_64-allyesdebian/4c3bffc272755c98728c2b58b1a8148cf9e9fd1f/dmesg-yocto-inn-17:20131008190244:x86_64-allyesdebian:3.11.0-09198-g4c3bffc:427 0:17:17 all_good:bad:all_bad boots REPEAT COUNT: 20 # /cc/wfg/mm-bisect/.repeat Bisecting: 4 revisions left to test after this (roughly 2 steps) [6e543d5780e36ff5ee56c44d7e2e30db3457a7ed] mm: vmscan: fix do_try_to_free_pages() livelock running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:6e543d5780e36ff5ee56c44d7e2e30db3457a7ed:bisect-mm 2013-10-08-19:02:53 6e543d5780e36ff5ee56c44d7e2e30db3457a7ed compiling 207 real 1131 user 90 sys 588.08% cpu x86_64-allyesdebian 2013-10-08-19:08:42 detecting boot state 3.11.0-09193-g6e543d5. TEST FAILURE [ 4.465347] pci 0000:00:02.0: Boot video device [ 4.466841] PCI: CLS 0 bytes, default 64 [ 5.010202] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 5.012753] swapper/0 cpuset=/ mems_allowed=0 [ 5.014121] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.11.0-09193-g6e543d5 #428 [ 5.016504] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 5.018184] 0000000000000002 ffff88001ef79b18 ffffffff82c5582d ffff88001ef74040 [ 5.021124] ffff88001ef79b98 ffffffff82c4dcb6 ffffffff83c54580 ffff88001ef79b50 [ 5.024153] ffffffff810ee06e 0000000000001a3a 0000000000000246 ffff88001ef79b88 [ 5.027012] Call Trace: [ 5.027980] [] dump_stack+0x54/0x74 [ 5.029544] [] dump_header.isra.10+0x7a/0x1ba [ 5.031267] [] ? lock_release_holdtime.part.27+0x4c/0x50 [ 5.034019] [] ? lock_release+0x189/0x1d1 [ 5.035644] [] out_of_memory+0x39e/0x3ee [ 5.037325] [] __alloc_pages_nodemask+0x668/0x7de [ 5.039117] [] kmem_getpages+0x75/0x16c [ 5.040760] [] fallback_alloc+0x12c/0x1ea [ 5.042491] [] ? trace_hardirqs_off+0xd/0xf [ 5.044258] [] ____cache_alloc_node+0x14a/0x159 [ 5.046063] [] __kmalloc+0x95/0x12d [ 5.047577] [] ? kzalloc.constprop.16+0xe/0x10 [ 5.049369] [] kzalloc.constprop.16+0xe/0x10 [ 5.051053] [] dma_debug_init+0x1e3/0x28e [ 5.052769] [] pci_iommu_init+0x16/0x52 [ 5.054370] [] ? iommu_setup+0x27d/0x27d [ 5.055986] [] do_one_initcall+0x93/0x137 [ 5.057662] [] ? parameq+0xd/0x24 [ 5.059155] [] ? parse_args+0x189/0x247 [ 5.060829] [] kernel_init_freeable+0x15e/0x1df [ 5.062610] [] ? do_early_param+0x88/0x88 [ 5.064274] [] ? rest_init+0xdb/0xdb [ 5.065814] [] kernel_init+0xe/0xdb [ 5.067356] [] ret_from_fork+0x7c/0xb0 [ 5.068992] [] ? rest_init+0xdb/0xdb [ 5.070556] Mem-Info: [ 5.071467] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-nfsroot-waimea-9:20131009030539:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-quantal-waimea-11:20131009030542:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-yocto-waimea-3:20131009030542:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-quantal-waimea-13:20131009030545:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-yocto-nhm4-3:20131008111351:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-yocto-snb-38:20131008190853:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-nfsroot-roam-12:20131008190858:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-nfsroot-waimea-9:20131009030548:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-quantal-waimea-11:20131009030554:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-quantal-xps-3:20131009030516:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-yocto-xps-1:20131009030517:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-quantal-bens-3:20131008190902:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-quantal-lkp-st01-1:20131008190901:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 /kernel/x86_64-allyesdebian/6e543d5780e36ff5ee56c44d7e2e30db3457a7ed/dmesg-quantal-lkp-tt02-24:20131008094002:x86_64-allyesdebian:3.11.0-09193-g6e543d5:428 0:14:14 all_good:bad:all_bad boots REPEAT COUNT: 20 # /cc/wfg/mm-bisect/.repeat Bisecting: 2 revisions left to test after this (roughly 1 step) [56afe477df3cbbcd656682d0355ef7d9eb8bdd81] mm: munlock: bypass per-cpu pvec for putback_lru_page running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:56afe477df3cbbcd656682d0355ef7d9eb8bdd81:bisect-mm 2013-10-08-19:09:20 56afe477df3cbbcd656682d0355ef7d9eb8bdd81 compiling 897 real 4902 user 426 sys 593.89% cpu x86_64-allyesdebian 2013-10-08-19:25:35 detecting boot state 3.11.0-09190-g56afe47.................... 3 20 SUCCESS Bisecting: 0 revisions left to test after this (roughly 1 step) [7a8010cd36273ff5f6fea5201ef9232f30cebbd9] mm: munlock: manual pte walk in fast path instead of follow_page_mask() running /c/kernel-tests/bisect-test-boot-failure.sh /home/wfg/mm/obj-bisect ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:7a8010cd36273ff5f6fea5201ef9232f30cebbd9:bisect-mm 2013-10-08-19:36:37 7a8010cd36273ff5f6fea5201ef9232f30cebbd9 compiling 916 real 4336 user 368 sys 513.49% cpu x86_64-allyesdebian 2013-10-08-19:52:25 detecting boot state 3.11.0-09192-g7a8010c.................... 15 19 20 SUCCESS 6e543d5780e36ff5ee56c44d7e2e30db3457a7ed is the first bad commit commit 6e543d5780e36ff5ee56c44d7e2e30db3457a7ed Author: Lisa Du Date: Wed Sep 11 14:22:36 2013 -0700 mm: vmscan: fix do_try_to_free_pages() livelock This patch is based on KOSAKI's work and I add a little more description, please refer https://lkml.org/lkml/2012/6/14/74. Currently, I found system can enter a state that there are lots of free pages in a zone but only order-0 and order-1 pages which means the zone is heavily fragmented, then high order allocation could make direct reclaim path's long stall(ex, 60 seconds) especially in no swap and no compaciton enviroment. This problem happened on v3.4, but it seems issue still lives in current tree, the reason is do_try_to_free_pages enter live lock: kswapd will go to sleep if the zones have been fully scanned and are still not balanced. As kswapd thinks there's little point trying all over again to avoid infinite loop. Instead it changes order from high-order to 0-order because kswapd think order-0 is the most important. Look at 73ce02e9 in detail. If watermarks are ok, kswapd will go back to sleep and may leave zone->all_unreclaimable =3D 0. It assume high-order users can still perform direct reclaim if they wish. Direct reclaim continue to reclaim for a high order which is not a COSTLY_ORDER without oom-killer until kswapd turn on zone->all_unreclaimble= . This is because to avoid too early oom-kill. So it means direct_reclaim depends on kswapd to break this loop. In worst case, direct-reclaim may continue to page reclaim forever when kswapd sleeps forever until someone like watchdog detect and finally kill the process. As described in: http://thread.gmane.org/gmane.linux.kernel.mm/103737 We can't turn on zone->all_unreclaimable from direct reclaim path because direct reclaim path don't take any lock and this way is racy. Thus this patch removes zone->all_unreclaimable field completely and recalculates zone reclaimable state every time. Note: we can't take the idea that direct-reclaim see zone->pages_scanned directly and kswapd continue to use zone->all_unreclaimable. Because, it is racy. commit 929bea7c71 (vmscan: all_unreclaimable() use zone->all_unreclaimable as a name) describes the detail. [akpm@linux-foundation.org: uninline zone_reclaimable_pages() and zone_reclaimable()] Cc: Aaditya Kumar Cc: Ying Han Cc: Nick Piggin Acked-by: Rik van Riel Cc: Mel Gorman Cc: KAMEZAWA Hiroyuki Cc: Christoph Lameter Cc: Bob Liu Cc: Neil Zhang Cc: Russell King - ARM Linux Reviewed-by: Michal Hocko Acked-by: Minchan Kim Acked-by: Johannes Weiner Signed-off-by: KOSAKI Motohiro Signed-off-by: Lisa Du Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds :040000 040000 9d60d44fa4168be4eb77e866dd27cd6d6f259879 ad9086448401d7792fe1c030ed60b30b306ba176 M include :040000 040000 951862cab689a62cd9b880bf1692801070d736f6 b059e9bc7179a409bac5ee1017e31e3494ab2c6c M mm bisect run success ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:7a8010cd36273ff5f6fea5201ef9232f30cebbd9:bisect-mm 2013-10-08-20:03:59 7a8010cd36273ff5f6fea5201ef9232f30cebbd9 reuse /kernel/x86_64-allyesdebian/7a8010cd36273ff5f6fea5201ef9232f30cebbd9/vmlinuz-3.11.0-09192-g7a8010c 2013-10-08-20:03:59 detecting boot state ... 1 2............... 13 35 50 56.. 57. 58.............................................................................................................................................................................................................................................................................................................add_to_run_queue 2 .................... 60 SUCCESS ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:8b5ede69d24db939f52b47effff2f6fe1e83e08b:bisect-mm TEST FAILURE [ 1.372082] pci 0000:00:02.0: Boot video device [ 1.373016] PCI: CLS 0 bytes, default 64 [ 1.576331] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 1.577804] swapper/0 cpuset=/ mems_allowed=0 [ 1.578614] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.12.0-rc4-00019-g8b5ede6 #126 [ 1.580044] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 1.581032] 0000000000000002 ffff88001dd41b28 ffffffff82c8d78f ffff88001ef7c040 [ 1.582731] ffff88001dd41ba8 ffffffff82c8395f ffffffff83c54680 ffff88001dd41b60 [ 1.584456] ffffffff810f3f06 0000000000001908 0000000000000246 ffff88001dd41b98 [ 1.586162] Call Trace: [ 1.586740] [] dump_stack+0x54/0x74 [ 1.587646] [] dump_header.isra.10+0x7a/0x1ba [ 1.588689] [] ? lock_release_holdtime.part.27+0x4c/0x50 [ 1.589829] [] ? lock_release+0x189/0x1d1 [ 1.590807] [] out_of_memory+0x39e/0x3ee [ 1.591777] [] __alloc_pages_nodemask+0x668/0x7de [ 1.592873] [] kmem_getpages+0x75/0x16c [ 1.593809] [] fallback_alloc+0x12c/0x1ea [ 1.594778] [] ? trace_hardirqs_off+0xd/0xf [ 1.595782] [] ____cache_alloc_node+0x14a/0x159 [ 1.596842] [] ? dma_debug_init+0x1ef/0x29a [ 1.597831] [] kmem_cache_alloc_trace+0x83/0x11a [ 1.598878] [] dma_debug_init+0x1ef/0x29a [ 1.599837] [] pci_iommu_init+0x16/0x52 [ 1.600815] [] ? iommu_setup+0x27d/0x27d [ 1.601778] [] do_one_initcall+0x93/0x137 [ 1.602750] [] ? param_set_charp+0x92/0xd8 [ 1.603728] [] ? parse_args+0x189/0x247 [ 1.604699] [] kernel_init_freeable+0x15e/0x1df [ 1.605738] [] ? do_early_param+0x88/0x88 [ 1.606703] [] ? rest_init+0xdb/0xdb [ 1.607616] [] kernel_init+0xe/0xdb [ 1.608541] [] ret_from_fork+0x7c/0xb0 [ 1.609478] [] ? rest_init+0xdb/0xdb [ 1.610385] Mem-Info: [ 1.610918] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-quantal-cairo-31:20131008141639:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-lkp-tt02-2:20131008044739:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-snb-22:20131008110042:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-athens-14:20131008141639:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-lkp-tt02-14:20131008044741:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-cairo-19:20131008110046:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-20:20131008141638:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-8:20131008110043:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-athens-4:20131008141643:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-quantal-athens-2:20131008110043:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-15:20131008110047:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-yocto-lkp-tt02-18:20131008013143:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-25:20131008110050:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-inn-28:20131008141639:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-23:20131008141636:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-xian-11:20131008141642:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-athens-30:20131008110043:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-yocto-snb-31:20131008141643:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-29:20131008110046:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 0:19:19 all_good:bad:all_bad boots [detached HEAD 880bdc8] Revert "mm: vmscan: fix do_try_to_free_pages() livelock" 9 files changed, 42 insertions(+), 44 deletions(-) ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:880bdc8e54a3b190a0dadb66a5f85e7c935eeb2b:bisect-mm 2013-10-08-23:00:02 880bdc8e54a3b190a0dadb66a5f85e7c935eeb2b compiling 2013-10-08-23:25:45 detecting boot state 3.12.0-rc4-00020-g880bdc8.................... 11 56 60 SUCCESS ========= upstream ========= Fetching linus ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:8b5ede69d24db939f52b47effff2f6fe1e83e08b:bisect-mm TEST FAILURE [ 1.372082] pci 0000:00:02.0: Boot video device [ 1.373016] PCI: CLS 0 bytes, default 64 [ 1.576331] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 1.577804] swapper/0 cpuset=/ mems_allowed=0 [ 1.578614] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.12.0-rc4-00019-g8b5ede6 #126 [ 1.580044] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 1.581032] 0000000000000002 ffff88001dd41b28 ffffffff82c8d78f ffff88001ef7c040 [ 1.582731] ffff88001dd41ba8 ffffffff82c8395f ffffffff83c54680 ffff88001dd41b60 [ 1.584456] ffffffff810f3f06 0000000000001908 0000000000000246 ffff88001dd41b98 [ 1.586162] Call Trace: [ 1.586740] [] dump_stack+0x54/0x74 [ 1.587646] [] dump_header.isra.10+0x7a/0x1ba [ 1.588689] [] ? lock_release_holdtime.part.27+0x4c/0x50 [ 1.589829] [] ? lock_release+0x189/0x1d1 [ 1.590807] [] out_of_memory+0x39e/0x3ee [ 1.591777] [] __alloc_pages_nodemask+0x668/0x7de [ 1.592873] [] kmem_getpages+0x75/0x16c [ 1.593809] [] fallback_alloc+0x12c/0x1ea [ 1.594778] [] ? trace_hardirqs_off+0xd/0xf [ 1.595782] [] ____cache_alloc_node+0x14a/0x159 [ 1.596842] [] ? dma_debug_init+0x1ef/0x29a [ 1.597831] [] kmem_cache_alloc_trace+0x83/0x11a [ 1.598878] [] dma_debug_init+0x1ef/0x29a [ 1.599837] [] pci_iommu_init+0x16/0x52 [ 1.600815] [] ? iommu_setup+0x27d/0x27d [ 1.601778] [] do_one_initcall+0x93/0x137 [ 1.602750] [] ? param_set_charp+0x92/0xd8 [ 1.603728] [] ? parse_args+0x189/0x247 [ 1.604699] [] kernel_init_freeable+0x15e/0x1df [ 1.605738] [] ? do_early_param+0x88/0x88 [ 1.606703] [] ? rest_init+0xdb/0xdb [ 1.607616] [] kernel_init+0xe/0xdb [ 1.608541] [] ret_from_fork+0x7c/0xb0 [ 1.609478] [] ? rest_init+0xdb/0xdb [ 1.610385] Mem-Info: [ 1.610918] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-quantal-cairo-31:20131008141639:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-lkp-tt02-2:20131008044739:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-snb-22:20131008110042:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-athens-14:20131008141639:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-lkp-tt02-14:20131008044741:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-cairo-19:20131008110046:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-20:20131008141638:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-8:20131008110043:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-athens-4:20131008141643:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-quantal-athens-2:20131008110043:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-15:20131008110047:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-yocto-lkp-tt02-18:20131008013143:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-25:20131008110050:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-inn-28:20131008141639:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-23:20131008141636:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-xian-11:20131008141642:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-athens-30:20131008110043:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-yocto-snb-31:20131008141643:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 /kernel/x86_64-allyesdebian/8b5ede69d24db939f52b47effff2f6fe1e83e08b/dmesg-nfsroot-roam-29:20131008110046:x86_64-allyesdebian:3.12.0-rc4-00019-g8b5ede6:126 0:19:19 all_good:bad:all_bad boots ========= linux-next ========= Fetching next ls -a /kernel-tests/run-queue/kvm/x86_64-allyesdebian/linus:master:a0cf1abc25ac197dd97b857c0f6341066a8cb1cf:bisect-mm TEST FAILURE [ 2.523706] pci 0000:00:02.0: Boot video device [ 2.526069] PCI: CLS 0 bytes, default 64 [ 2.990897] swapper/0 invoked oom-killer: gfp_mask=0x2000d0, order=1, oom_score_adj=0 [ 2.996176] swapper/0 cpuset=/ mems_allowed=0 [ 2.998239] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 3.12.0-rc2-next-20130927-03100-ga0cf1ab #482 [ 3.002383] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 3.004711] 0000000000000002 ffff88001ef7fb28 ffffffff82c8834f ffff88001dd40040 [ 3.008847] ffff88001ef7fba8 ffffffff82c7ec33 ffffffff83c54940 ffff88001ef7fb60 [ 3.012898] ffffffff810f3daa 00000000000035e4 0000000000000246 ffff88001ef7fb98 [ 3.017105] Call Trace: [ 3.018602] [] dump_stack+0x4d/0x66 [ 3.020752] [] dump_header.isra.10+0x7a/0x1ba [ 3.023295] [] ? lock_release_holdtime.part.27+0x71/0x7d [ 3.026278] [] ? lock_release+0x18c/0x1d7 [ 3.029472] [] out_of_memory+0x39e/0x3ee [ 3.032441] [] __alloc_pages_nodemask+0x659/0x7d5 [ 3.035697] [] kmem_getpages+0x75/0x16c [ 3.038245] [] fallback_alloc+0x12c/0x1ea [ 3.041008] [] ? trace_hardirqs_off+0xd/0xf [ 3.043268] [] ____cache_alloc_node+0x14a/0x159 [ 3.046156] [] ? dma_debug_init+0x1ef/0x29a [ 3.048842] [] kmem_cache_alloc_trace+0x83/0x11a [ 3.052081] [] dma_debug_init+0x1ef/0x29a [ 3.054295] [] pci_iommu_init+0x16/0x52 [ 3.057589] [] ? iommu_setup+0x27d/0x27d [ 3.061707] [] do_one_initcall+0x93/0x139 [ 3.065961] [] ? parameqn+0x21/0x3b [ 3.069936] [] ? parse_args+0x189/0x247 [ 3.073973] [] kernel_init_freeable+0x15e/0x1e4 [ 3.077805] [] ? do_early_param+0x88/0x88 [ 3.081895] [] ? rest_init+0xcd/0xcd [ 3.085769] [] kernel_init+0xe/0xdb [ 3.089012] [] ret_from_fork+0x7c/0xb0 [ 3.091525] [] ? rest_init+0xcd/0xcd [ 3.094035] Mem-Info: [ 3.095208] Node 0 DMA per-cpu: /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-nfsroot-waimea-5:20130928183212:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-quantal-ant-2:20130928103415:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-nfsroot-waimea-3:20130928183343:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-yocto-lkp-tt02-21:20130928010336:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-quantal-waimea-12:20130928183817:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-quantal-waimea-7:20130928183811:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-nfsroot-waimea-3:20130928183358:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-quantal-lkp-tt02-22:20130928010354:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 /kernel/x86_64-allyesdebian/a0cf1abc25ac197dd97b857c0f6341066a8cb1cf/dmesg-quantal-waimea-12:20130928183800:3.12.0-rc2-next-20130927-03100-ga0cf1ab:482 0:9:9 all_good:bad:all_bad boots