Hi. We're seeing such a deadlock, seems to be VM related very often. Servers are quite loaded, this is FreeBSD 6.2 on SMP Opteron system, but running i386 FreeBSD. Here is some info from one of such machines: db> ps pid ppid pgrp uid state wmesg wchan cmd 90346 4316 4316 101 L *Giant 0xc8551ac0 qmail-smtpd [...] 89997 4316 4316 101 L *vm page 0xcabc9600 qmail-smtpd [...] 4316 4311 4316 101 LL *vm page 0xcabc9600 qmail-smtpd 4315 4311 4315 101 RL drwebd [...] 34 0 0 0 LL *vm page 0xcabc9600 [pagezero] [...] 16 0 0 0 LL *vm page 0xcabc9600 [swi1: net] [...] 14 0 0 0 LL *tcp 0xcabc82c0 [swi4: clock sio] [...] 3 0 0 0 LL *vm page 0xcabc9600 [g_up] db> alltrace Tracing command qmail-smtpd pid 90346 tid 100260 td 0xc8c12a80 sched_switch(c8c12a80,0,1) at sched_switch+0x14b mi_switch(1,0) at mi_switch+0x1ba turnstile_wait(c07177e0,cb62d000) at turnstile_wait+0x2f7 _mtx_lock_sleep(c07177e0,c8c12a80,0,0,0) at _mtx_lock_sleep+0xfc vn_write(cac3b288,f84c9cbc,cb595000,0,c8c12a80) at vn_write+0x57 dofilewrite(c8c12a80,3,cac3b288,f84c9cbc,ffffffff,...) at dofilewrite+0x77 kern_writev(c8c12a80,3,f84c9cbc,bfbfe7f0,8c,...) at kern_writev+0x3b write(c8c12a80,f84c9d04) at write+0x45 syscall(83f003b,80c003b,bfbf003b,0,bfbfecbc,...) at syscall+0x2bf Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (4, FreeBSD ELF32, write), eip = 0x80a5def, esp = 0xbfbfe6cc, ebp = 0xbfbfec08 --- Tracing command qmail-smtpd pid 89997 tid 100133 td 0xcb62d000 sched_switch(cb62d000,0,1) at sched_switch+0x14b mi_switch(1,0) at mi_switch+0x1ba turnstile_wait(c0722700,cac19600) at turnstile_wait+0x2f7 _mtx_lock_sleep(c0722700,cb62d000,0,0,0) at _mtx_lock_sleep+0xfc vfs_clean_pages(e84b5f98,e84b5f98,25,f82a59f4,c83f2300,...) at vfs_clean_pages+0x8f bdwrite(e84b5f98) at bdwrite+0x338 cluster_write(c8a49cc0,e84b5f98,88000,0,25) at cluster_write+0x51f ffs_write(f82a5bec) at ffs_write+0x508 VOP_WRITE_APV(c06fe400,f82a5bec) at VOP_WRITE_APV+0xce vn_write(c9f91af8,f82a5cbc,cb595000,0,cb62d000) at vn_write+0x1ee dofilewrite(cb62d000,5,c9f91af8,f82a5cbc,ffffffff,...) at dofilewrite+0x77 kern_writev(cb62d000,5,f82a5cbc,80e2ed8,7d25,...) at kern_writev+0x3b write(cb62d000,f82a5d04) at write+0x45 syscall(bfbf003b,bfbf003b,bfbf003b,bfbfe37c,bfbfecbc,...) at syscall+0x2bf Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (4, FreeBSD ELF32, write), eip = 0x80a5def, esp = 0xbfbfe2cc, ebp = 0xbfbfe2e8 --- Tracing command qmail-smtpd pid 4316 tid 100098 td 0xc9f84d80 sched_switch(c9f84d80,0,1) at sched_switch+0x14b mi_switch(1,0) at mi_switch+0x1ba turnstile_wait(c0722700,cac19600) at turnstile_wait+0x2f7 _mtx_lock_sleep(c0722700,c9f84d80,0,0,0) at _mtx_lock_sleep+0xfc vm_fault(c83f8a68,80e2000,2,8) at vm_fault+0x2ca trap_pfault(f81b9c40,0,80e29e4) at trap_pfault+0x103 trap(c05c0008,28,c06f0028,80e29e4,c855b160,...) at trap+0x2fd calltrap() at calltrap+0x5 --- trap 0xc, eip = 0xc066b056, esp = 0xf81b9c80, ebp = 0xf81b9cc8 --- generic_copyout(c9f84d80,f81b9d04,0,f81b9d30,c066d47b,...) at generic_copyout+0x36 accept(c9f84d80,f81b9d04) at accept+0x10 syscall(83f003b,80c003b,bfbf003b,0,bfbfecbc,...) at syscall+0x2bf Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (30, FreeBSD ELF32, accept), eip = 0x8074e5b, esp = 0xbfbfec1c, ebp = 0xbfbfec48 --- Tracing command drwebd pid 4315 tid 100105 td 0xcac19600 sched_switch(cac19600,0,2) at sched_switch+0x14b mi_switch(2,0) at mi_switch+0x1ba critical_exit(0,cac19600,ca0b5688,8246000,0,...) at critical_exit+0x9d intr_execute_handlers(c844b0c8,f8211b90,34,f8211be0,c06590b3,...) at intr_execute_handlers+0x10c lapic_handle_intr(31) at lapic_handle_intr+0x2e Xapic_isr1() at Xapic_isr1+0x33 --- interrupt, eip = 0xc06676c4, esp = 0xf8211bd4, ebp = 0xf8211be0 --- pmap_pte_quick(ca0b5688,8246000) at pmap_pte_quick+0x90 pmap_copy(ca0b5688,c88d1ea0,81a1000,a18000,81a1000) at pmap_copy+0x27c vm_map_copy_entry(c88d1de0,ca0b55c8,cabc4374,cac161dc) at vm_map_copy_entry+0x14c vmspace_fork(c88d1de0,4,c854b860,c8823d00,f8211cbc,...) at vmspace_fork+0x22a vm_forkproc(cac19600,c854b860,c854c900,14) at vm_forkproc+0xb3 fork1(cac19600,14,0,f8211cd4) at fork1+0x1169 fork(cac19600,f8211d04) at fork+0x18 syscall(3b,3b,bfbf003b,4,81bba00,...) at syscall+0x2bf Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (2, FreeBSD ELF32, fork), eip = 0x80e493f, esp = 0xbfbfdcac, ebp = 0xbfbfe148 --- Tracing command pagezero pid 34 tid 100031 td 0xc83f6a80 sched_switch(c83f6a80,0,1) at sched_switch+0x14b mi_switch(1,0) at mi_switch+0x1ba turnstile_wait(c0722700,cac19600) at turnstile_wait+0x2f7 _mtx_lock_sleep(c0722700,c83f6a80,0,0,0) at _mtx_lock_sleep+0xfc vm_pagezero(0,f3a06d38) at vm_pagezero+0x4f fork_exit(c061d22c,0,f3a06d38) at fork_exit+0x71 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xf3a06d6c, ebp = 0 --- Tracing command swi1: net pid 16 tid 100000 td 0xc83f2000 sched_switch(c83f2000,0,1) at sched_switch+0x14b mi_switch(1,0) at mi_switch+0x1ba turnstile_wait(c0722700,cac19600) at turnstile_wait+0x2f7 _mtx_lock_sleep(c0722700,c83f2000,0,0,0) at _mtx_lock_sleep+0xfc sf_buf_mext(f4735000,c84211c0) at sf_buf_mext+0x42 mb_free_ext(c898db00) at mb_free_ext+0xd6 sbdrop_locked(c8894658,517,0,0,1,...) at sbdrop_locked+0xc0 tcp_input(cec2ba00,14,c0551efd,2,c055193f,...) at tcp_input+0x1ec6 ip_input(cec2ba00) at ip_input+0x5ed netisr_processqueue(c07202d8) at netisr_processqueue+0xa7 swi_net(0) at swi_net+0xf6 ithread_execute_handlers(c83f0000,c8459e00) at ithread_execute_handlers+0x125 ithread_loop(c83d4850,f39d3d38) at ithread_loop+0x55 fork_exit(c0529814,c83d4850,f39d3d38) at fork_exit+0x71 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xf39d3d6c, ebp = 0 --- Tracing command swi4: clock sio pid 14 tid 100002 td 0xc83f1c00 sched_switch(c83f1c00,0,1) at sched_switch+0x14b mi_switch(1,0) at mi_switch+0x1ba turnstile_wait(c072146c,c83f2000) at turnstile_wait+0x2f7 _mtx_lock_sleep(c072146c,c83f1c00,0,0,0) at _mtx_lock_sleep+0xfc tcp_timer_delack(c9723ae0) at tcp_timer_delack+0x37 softclock(0) at softclock+0x2eb ithread_execute_handlers(c83f0430,c845a000) at ithread_execute_handlers+0x125 ithread_loop(c83d4870,f39cdd38) at ithread_loop+0x55 fork_exit(c0529814,c83d4870,f39cdd38) at fork_exit+0x71 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xf39cdd6c, ebp = 0 --- Tracing command g_up pid 3 tid 100015 td 0xc83f2c00 sched_switch(c83f2c00,0,1) at sched_switch+0x14b mi_switch(1,0) at mi_switch+0x1ba turnstile_wait(c0722700,cac19600) at turnstile_wait+0x2f7 _mtx_lock_sleep(c0722700,c83f2c00,0,0,0) at _mtx_lock_sleep+0xfc bufdone(e8591a10) at bufdone+0x12b g_vfs_done(c8862738) at g_vfs_done+0xab biodone(c8862738) at biodone+0x8b g_io_schedule_up(c83f2c00) at g_io_schedule_up+0x86 g_up_procbody(0,f39ebd38) at g_up_procbody+0x92 fork_exit(c04fcb78,0,f39ebd38) at fork_exit+0x71 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xf39ebd6c, ebp = 0 --- db> sh lockchain thread 100003 (pid 13, idle: cpu0) running on CPU 0 db> sh allchain chain 1: thread 100260 (pid 90346, qmail-smtpd) blocked on lock 0xc07177e0 (sleep mutex) "Giant" thread 100133 (pid 89997, qmail-smtpd) blocked on lock 0xc0722700 (sleep mutex) "vm page queue mutex" thread 100105 (pid 4315, drwebd) is on a run queue chain 2: thread 100098 (pid 4316, qmail-smtpd) blocked on lock 0xc0722700 (sleep mutex) "vm page queue mutex" thread 100105 (pid 4315, drwebd) is on a run queue chain 3: thread 100031 (pid 34, pagezero) blocked on lock 0xc0722700 (sleep mutex) "vm page queue mutex" thread 100105 (pid 4315, drwebd) is on a run queue chain 4: thread 100015 (pid 3, g_up) blocked on lock 0xc0722700 (sleep mutex) "vm page queue mutex" thread 100105 (pid 4315, drwebd) is on a run queue chain 5: thread 100002 (pid 14, swi4: clock sio) blocked on lock 0xc072146c (sleep mutex) "tcp" thread 100000 (pid 16, swi1: net) blocked on lock 0xc0722700 (sleep mutex) "vm page queue mutex" thread 100105 (pid 4315, drwebd) is on a run queue db> sh locktree db> sh turnstile Process drwebd (pid 4315) doesn't seem to go any further. -- Pawel Jakub Dawidek http://www.wheel.pl pjd_at_FreeBSD.org http://www.FreeBSD.org FreeBSD committer Am I Evil? Yes, I Am!
This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:39:02 UTC