(another) panic on shutdown (spin lock held too long)

From: Thierry Herbelot <thierry_at_herbelot.com>
Date: Fri, 5 Sep 2008 07:06:01 +0200
Hello,

I was just rebooting a recent SMP -current machine, running a straight GENERIC 
kernel, with only / mounted read-only, and I got the following panic :

	TfH

# mount
/dev/mirror/gm0a on / (ufs, local, read-only)
devfs on /dev (devfs, local)
# reboot
Waiting (max 60 seconds) for system process `vnlru' to stop...done
Waiting (max 60 seconds) for system process `bufdaemon' to stop...done
Waiting (max 60 seconds) for system process `syncer' to stop...
Syncing disks, vnodes remaining...0 0 0 0 0 0 0 0 0 done
All buffers synced.
Uptime: 11m36s
GEOM_MIRROR: Device gm0: provider mirror/gm0 destroyed.
Rebooting...
cpu_reset: Stopping other CPUs
spin lock 0xc0cb4e00 (sched lock 1) held by 0xc2cc2d20 (tid 100002) too long
panic: spin lock held too long
cpuid = 0
KDB: enter: panic
[thread pid 34 tid 100056 ]
Stopped at      kdb_enter+0x3a: movl    $0,kdb_why
db> where
Tracing pid 34 tid 100056 td 0xc2fbe000
kdb_enter(c0b6aaea,c0b6aaea,c0b697e9,e011ca58,0,...) at kdb_enter+0x3a
panic(c0b697e9,c2cc2d20,c0cb5448,c2cc2d20,186a2,...) at panic+0x12c
_mtx_lock_spin_failed(1,c2fbe000,c0cb4e00,10,e011cab0,...) at 
_mtx_lock_spin_failed+0x51
_mtx_lock_spin(c0cb4e00,c2fbe000,10,c0b6c6e7,316,...) at _mtx_lock_spin+0x75
_mtx_lock_spin_flags(c0cb4e00,10,c0b6c6e7,316,c0cbc760,...) at 
_mtx_lock_spin_flags+0x13e
tdq_lock_pair(c0cbc760,e011cafc,e011cb0c,c0cbc760,ffffffff,...) at 
tdq_lock_pair+0x54
sched_balance_group(c0cb4780,0,c0b6c6e7,309,0,...) at sched_balance_group+0xba
sched_balance(c0cb4780,4,c0b6c6e7,839,0,...) at sched_balance+0x8a
sched_clock(c2fbe000,2,c0b65818,1f7,c0e31320,...) at sched_clock+0x6b
statclock(0,c0acc759,0,40,49e57b9c,...) at statclock+0x11e
lapic_handle_timer(e011cbb0) at lapic_handle_timer+0x101
Xtimerint() at Xtimerint+0x1f
--- interrupt, eip = 0xc0acc75b, esp = 0xe011cbf0, ebp = 0xe011cc0c ---
DELAY(f4240,0,c2ca9960,e011cc2c,c07b2ab3,...) at DELAY+0x8b
cpu_reset(f4240,e011cc68,c07b358f,0,0,...) at cpu_reset+0xd5
shutdown_reset(0,0,c0b6a99c,1a5,0,...) at shutdown_reset+0x23
boot(c0cb0630,0,c0b6a99c,ab,e011cd2c,...) at boot+0x75f
reboot(c2fbe000,e011ccf8,4,c0b71f60,c0c40d48,...) at reboot+0x4b
syscall(e011cd38) at syscall+0x2a3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (55, FreeBSD ELF32, reboot), eip = 0x280be56b, esp = 0xbfbfee1c, 
ebp = 0xbfbfee58 ---
db> show locks;
exclusive sleep mutex Giant (Giant) r = 0 (0xc0cb0630) locked 
_at_ /tank/files1/src/sys/kern/kern_module.c:102
db> show alllocks;
Process 34 (reboot) thread 0xc2fbe000 (100056)
exclusive sleep mutex Giant (Giant) r = 0 (0xc0cb0630) locked 
_at_ /tank/files1/src/sys/kern/kern_module.c:102
Process 26 (softdepflush) thread 0xc2deeaf0 (100048)
Process 25 (vnlru) thread 0xc2deed20 (100047)
Process 24 (syncer) thread 0xc2e4a000 (100046)
Process 23 (bufdaemon) thread 0xc2e4a230 (100045)
Process 22 (pagezero) thread 0xc2e4a460 (100044)
Process 21 (vmdaemon) thread 0xc2e4a690 (100043)
Process 9 (pagedaemon) thread 0xc2e4a8c0 (100042)
Process 7 (sctp_iterator) thread 0xc2e4ad20 (100040)
Process 6 (fdc0) thread 0xc2ded230 (100036)
Process 20 (usb4) thread 0xc2ded8c0 (100033)
Process 19 (usb3) thread 0xc2dedd20 (100031)
Process 18 (usb2) thread 0xc2dee230 (100029)
Process 17 (usb1) thread 0xc2d07690 (100027)
Process 16 (usbtask-dr) thread 0xc2d078c0 (100026)
Process 15 (usbtask-hc) thread 0xc2d07af0 (100025)
Process 14 (usb0) thread 0xc2d07d20 (100024)
Process 5 (xpt_thrd) thread 0xc2dc4af0 (100018)
Process 13 (yarrow) thread 0xc2cc4af0 (100013)
Process 4 (g_down) thread 0xc2cc4d20 (100012)
Process 3 (g_up) thread 0xc2d07000 (100011)
Process 2 (g_event) thread 0xc2d07230 (100010)
exclusive sx gmirror:lock (gmirror:lock) r = 0 (0xc2e4762c) locked 
_at_ /tank/files1/src/sys/modules/geom/geom_mirror/../../../geom/mirror/g_mirror.c:2977
Process 12 (swi0: uart uart) thread 0xc2dc4d20 (100038)
Process 12 (irq15: ata1) thread 0xc2dc4230 (100022)
Process 12 (irq14: ata0) thread 0xc2dc4460 (100021)
Process 12 (swi6: task queue) thread 0xc2cc4460 (100016)
Process 12 (swi4: clock) thread 0xc2cc2690 (100005)
Process 1 (init) thread 0xc2cc2d20 (100002)
exclusive sleep mutex process lock (process lock) r = 0 (0xc2cc0d94) locked 
_at_ /tank/files1/src/sys/kern/kern_sig.c:1831
Process 10 (audit) thread 0xc2cc4000 (100001)
Process 0 (kqueue taskq) thread 0xc2cc4230 (100017)
Process 0 (thread taskq) thread 0xc2cc48c0 (100014)
Process 0 (firmware taskq) thread 0xc2d07460 (100009)
Process 0 (swapper) thread 0xc0caed20 (100000)
db> show lockedvnods
Locked vnodes
db> bt
Tracing pid 34 tid 100056 td 0xc2fbe000
kdb_enter(c0b6aaea,c0b6aaea,c0b697e9,e011ca58,0,...) at kdb_enter+0x3a
panic(c0b697e9,c2cc2d20,c0cb5448,c2cc2d20,186a2,...) at panic+0x12c
_mtx_lock_spin_failed(1,c2fbe000,c0cb4e00,10,e011cab0,...) at 
_mtx_lock_spin_failed+0x51
_mtx_lock_spin(c0cb4e00,c2fbe000,10,c0b6c6e7,316,...) at _mtx_lock_spin+0x75
_mtx_lock_spin_flags(c0cb4e00,10,c0b6c6e7,316,c0cbc760,...) at 
_mtx_lock_spin_flags+0x13e
tdq_lock_pair(c0cbc760,e011cafc,e011cb0c,c0cbc760,ffffffff,...) at 
tdq_lock_pair+0x54
sched_balance_group(c0cb4780,0,c0b6c6e7,309,0,...) at sched_balance_group+0xba
sched_balance(c0cb4780,4,c0b6c6e7,839,0,...) at sched_balance+0x8a
sched_clock(c2fbe000,2,c0b65818,1f7,c0e31320,...) at sched_clock+0x6b
statclock(0,c0acc759,0,40,49e57b9c,...) at statclock+0x11e
lapic_handle_timer(e011cbb0) at lapic_handle_timer+0x101
Xtimerint() at Xtimerint+0x1f
--- interrupt, eip = 0xc0acc75b, esp = 0xe011cbf0, ebp = 0xe011cc0c ---
DELAY(f4240,0,c2ca9960,e011cc2c,c07b2ab3,...) at DELAY+0x8b
cpu_reset(f4240,e011cc68,c07b358f,0,0,...) at cpu_reset+0xd5
shutdown_reset(0,0,c0b6a99c,1a5,0,...) at shutdown_reset+0x23
boot(c0cb0630,0,c0b6a99c,ab,e011cd2c,...) at boot+0x75f
reboot(c2fbe000,e011ccf8,4,c0b71f60,c0c40d48,...) at reboot+0x4b
syscall(e011cd38) at syscall+0x2a3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (55, FreeBSD ELF32, reboot), eip = 0x280be56b, esp = 0xbfbfee1c, 
ebp = 0xbfbfee58 ---
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
   34     1    34     0  R+      CPU 0               reboot
   26     0     0     0  SL      sdflush  0xc0e224e0 [softdepflush]
   25     0     0     0  SL      kpsusp   0xc2f21874 [vnlru]
   24     0     0     0  SL      kpsusp   0xc2d39874 [syncer]
   23     0     0     0  SL      kpsusp   0xc2d39b10 [bufdaemon]
   22     0     0     0  SL      pgzero   0xc0e230d4 [pagezero]
   21     0     0     0  SL      psleep   0xc0e22cfc [vmdaemon]
    9     0     0     0  SL      psleep   0xc0e22cc4 [pagedaemon]
    7     0     0     0  SL      waiting_ 0xc0e18280 [sctp_iterator]
    6     0     0     0  SL      -        0xc2dff23c [fdc0]
   20     0     0     0  SL      usbevt   0xc2dbf210 [usb4]
   19     0     0     0  SL      usbevt   0xc2df9210 [usb3]
   18     0     0     0  SL      usbevt   0xc2de0210 [usb2]
   17     0     0     0  SL      usbevt   0xc2de2210 [usb1]
   16     0     0     0  SL      usbtsk   0xc0cae294 [usbtask-dr]
   15     0     0     0  SL      usbtsk   0xc0cae280 [usbtask-hc]
   14     0     0     0  SL      usbevt   0xc2dd1210 [usb0]
    5     0     0     0  SL      ccb_scan 0xc0c7e054 [xpt_thrd]
   13     0     0     0  SL      -        0xc0cb0bc4 [yarrow]
    4     0     0     0  RL                          [g_down]
    3     0     0     0  SL      -        0xc0cae9c0 [g_up]
    2     0     0     0  RL                          [g_event]
   12     0     0     0  LL      (threaded)          intr
100039                   I                           [irq5: pcm1]
100038                   I                           [swi0: uart uart]
100037                   I                           [irq7: ppbus0 ppc0]
100035                   I                           [irq12: psm0]
100034                   I                           [irq1: atkbd0]
100032                   I                           [irq18: ehci0++]
100030                   I                           [irq17: ohci2]
100028                   I                           [irq16: ohci1]
100023                   I                           [irq19: pcm0 dc0++]
100022                   I                           [irq15: ata1]
100021                   I                           [irq14: ata0]
100020                   I                           [swi5: +]
100019                   I                           [swi2: cambio]
100016                   I                           [swi6: task queue]
100015                   I                           [swi6: Giant taskq]
100008                   I                           [swi3: vm]
100007                   I                           [swi1: net]
100006                   I                           [swi4: clock]
100005                   L      *Giant    0xc2c8b2d0 [swi4: clock]
   11     0     0     0  RL      (threaded)          idle
100004                   CanRun                      [idle: cpu0]
100003                   CanRun                      [idle: cpu1]
    1     0     1     0  RLs     CPU 1               [init]
   10     0     0     0  SL      audit_wo 0xc0e21ee0 [audit]
    0     0     0     0  SLs     (threaded)          kernel
100017                   D       -        0xc2db1140 [kqueue taskq]
100014                   D       -        0xc2db1280 [thread taskq]
100009                   D       -        0xc2ca2700 [firmware taskq]
100000                   D       sched    0xc0caea80 [swapper]
db> alltrace

Tracing command reboot pid 34 tid 100056 td 0xc2fbe000
kdb_enter(c0b6aaea,c0b6aaea,c0b697e9,e011ca58,0,...) at kdb_enter+0x3a
panic(c0b697e9,c2cc2d20,c0cb5448,c2cc2d20,186a2,...) at panic+0x12c
_mtx_lock_spin_failed(1,c2fbe000,c0cb4e00,10,e011cab0,...) at 
_mtx_lock_spin_failed+0x51
_mtx_lock_spin(c0cb4e00,c2fbe000,10,c0b6c6e7,316,...) at _mtx_lock_spin+0x75
_mtx_lock_spin_flags(c0cb4e00,10,c0b6c6e7,316,c0cbc760,...) at 
_mtx_lock_spin_flags+0x13e
tdq_lock_pair(c0cbc760,e011cafc,e011cb0c,c0cbc760,ffffffff,...) at 
tdq_lock_pair+0x54
sched_balance_group(c0cb4780,0,c0b6c6e7,309,0,...) at sched_balance_group+0xba
sched_balance(c0cb4780,4,c0b6c6e7,839,0,...) at sched_balance+0x8a
sched_clock(c2fbe000,2,c0b65818,1f7,c0e31320,...) at sched_clock+0x6b
statclock(0,c0acc759,0,40,49e57b9c,...) at statclock+0x11e
lapic_handle_timer(e011cbb0) at lapic_handle_timer+0x101
Xtimerint() at Xtimerint+0x1f
--- interrupt, eip = 0xc0acc75b, esp = 0xe011cbf0, ebp = 0xe011cc0c ---
DELAY(f4240,0,c2ca9960,e011cc2c,c07b2ab3,...) at DELAY+0x8b
cpu_reset(f4240,e011cc68,c07b358f,0,0,...) at cpu_reset+0xd5
shutdown_reset(0,0,c0b6a99c,1a5,0,...) at shutdown_reset+0x23
boot(c0cb0630,0,c0b6a99c,ab,e011cd2c,...) at boot+0x75f
reboot(c2fbe000,e011ccf8,4,c0b71f60,c0c40d48,...) at reboot+0x4b
syscall(e011cd38) at syscall+0x2a3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (55, FreeBSD ELF32, reboot), eip = 0x280be56b, esp = 0xbfbfee1c, 
ebp = 0xbfbfee58 ---

Tracing command softdepflush pid 26 tid 100048 td 0xc2deeaf0
sched_switch(c2deeaf0,0,104,17e,1fffe200,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,44,...) at mi_switch+0x200
sleepq_switch(c2deeaf0,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c0e224e0,44,c0b8db18,0,0,...) at sleepq_timedwait+0x6b
_sleep(c0e224e0,c0e22484,44,c0b8db18,3e8,...) at _sleep+0x2f9
softdep_flush(0,df41ad38,c0b66eb8,322,c2f21538,...) at softdep_flush+0x2b5
fork_exit(c09b0a60,0,df41ad38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf41ad70, ebp = 0 ---

Tracing command vnlru pid 25 tid 100047 td 0xc2deed20
sched_switch(c2deed20,0,104,17e,7217c14a,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,68,...) at mi_switch+0x200
sleepq_switch(c2deed20,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c2f21874,68,c0b676c7,0,0,...) at sleepq_wait+0x63
_sleep(c2f21874,c2f2185c,68,c0b676c7,0,...) at _sleep+0x32b
kproc_suspend_check(c2f217d4,c0e16870,250,c0b782a8,3e8,...) at 
kproc_suspend_check+0x7c
vnlru_proc(0,df417d38,c0b66eb8,322,c2f217d4,...) at vnlru_proc+0x68
fork_exit(c083bd80,0,df417d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf417d70, ebp = 0 ---

Tracing command syncer pid 24 tid 100046 td 0xc2e4a000
sched_switch(c2e4a000,0,104,17e,301bdf40,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,68,...) at mi_switch+0x200
sleepq_switch(c2e4a000,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c2d39874,68,c0b676c7,0,0,...) at sleepq_wait+0x63
_sleep(c2d39874,c2d3985c,68,c0b676c7,0,...) at _sleep+0x32b
kproc_suspend_check(c2d397d4,0,c0b77505,6ae,4e20,...) at 
kproc_suspend_check+0x7c
sched_sync(0,df414d38,c0b66eb8,322,c2d397d4,...) at sched_sync+0xdc
fork_exit(c083b200,0,df414d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf414d70, ebp = 0 ---

Tracing command bufdaemon pid 23 tid 100045 td 0xc2e4a230
sched_switch(c2e4a230,0,104,17e,7efee33c,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,68,...) at mi_switch+0x200
sleepq_switch(c2e4a230,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c2d39b10,68,c0b676c7,0,0,...) at sleepq_wait+0x63
_sleep(c2d39b10,c2d39af8,68,c0b676c7,0,...) at _sleep+0x32b
kproc_suspend_check(c2d39a70,0,c0b75392,7ed,3e8,...) at 
kproc_suspend_check+0x7c
buf_daemon(0,df411d38,c0b66eb8,322,c2d39a70,...) at buf_daemon+0xa0
fork_exit(c0824d40,0,df411d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf411d70, ebp = 0 ---

Tracing command pagezero pid 22 tid 100044 td 0xc2e4a460
sched_switch(c2e4a460,0,104,17e,12c8a303,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,0,...) at mi_switch+0x200
sleepq_switch(c2e4a460,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c0e230d4,0,c0b92c20,0,0,...) at sleepq_timedwait+0x6b
_sleep(c0e230d4,c0e22c90,0,c0b92c20,493e0,...) at _sleep+0x2f9
vm_pagezero(0,df40ed38,c0b66eb8,322,c2d39d0c,...) at vm_pagezero+0xdc
fork_exit(c09e9ce0,0,df40ed38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf40ed70, ebp = 0 ---

Tracing command vmdaemon pid 21 tid 100043 td 0xc2e4a690
sched_switch(c2e4a690,0,104,17e,3e7a2087,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,68,...) at mi_switch+0x200
sleepq_switch(c2e4a690,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c0e22cfc,68,c0b7598a,0,0,...) at sleepq_wait+0x63
_sleep(c0e22cfc,c0e22d00,68,c0b7598a,0,...) at _sleep+0x32b
vm_daemon(0,df40bd38,c0b66eb8,322,c2e01000,...) at vm_daemon+0x59
fork_exit(c09e4200,0,df40bd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf40bd70, ebp = 0 ---

Tracing command pagedaemon pid 9 tid 100042 td 0xc2e4a8c0
sched_switch(c2e4a8c0,0,104,17e,d97047cb,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,44,...) at mi_switch+0x200
sleepq_switch(c2e4a8c0,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c0e22cc4,44,c0b7598a,0,0,...) at sleepq_timedwait+0x6b
_sleep(c0e22cc4,c0e22c90,44,c0b7598a,1388,...) at _sleep+0x2f9
vm_pageout(0,df408d38,c0b66eb8,322,c2e0129c,...) at vm_pageout+0x2bb
fork_exit(c09e4d70,0,df408d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf408d70, ebp = 0 ---

Tracing command sctp_iterator pid 7 tid 100040 td 0xc2e4ad20
sched_switch(c2e4ad20,0,104,17e,1e9208b1,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,0,...) at mi_switch+0x200
sleepq_switch(c2e4ad20,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c0e18280,0,c0b813da,0,0,...) at sleepq_wait+0x63
_sleep(c0e18280,c0e18194,0,c0b813da,0,...) at _sleep+0x32b
sctp_iterator_thread(0,df402d38,c0b66eb8,322,c2e017d4,...) at 
sctp_iterator_thread+0x60
fork_exit(c08ae1f0,0,df402d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf402d70, ebp = 0 ---

Tracing command fdc0 pid 6 tid 100036 td 0xc2ded230
sched_switch(c2ded230,0,104,17e,2009f35a,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,4c,...) at mi_switch+0x200
sleepq_switch(c2ded230,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c2dff23c,4c,c0b61a9d,0,0,...) at sleepq_timedwait+0x6b
_sleep(c2dff23c,c2dff2f0,4c,c0b61a9d,3e8,...) at _sleep+0x2f9
fdc_thread(c2dff200,df3efd38,c0b66eb8,322,c2e01a70,...) at fdc_thread+0x2b8
fork_exit(c0a8a890,c2dff200,df3efd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3efd70, ebp = 0 ---

Tracing command usb4 pid 20 tid 100033 td 0xc2ded8c0
sched_switch(c2ded8c0,0,104,17e,f2da2b2f,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,5c,...) at mi_switch+0x200
sleepq_switch(c2ded8c0,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c2dbf210,5c,c0b5db5c,0,0,...) at sleepq_timedwait+0x6b
_sleep(c2dbf210,0,5c,c0b5db5c,ea60,...) at _sleep+0x2f9
usb_event_thread(c2def940,df3d4d38,c0b66eb8,322,c2e01d0c,...) at 
usb_event_thread+0xca
fork_exit(c071c160,c2def940,df3d4d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3d4d70, ebp = 0 ---

Tracing command usb3 pid 19 tid 100031 td 0xc2dedd20
sched_switch(c2dedd20,0,104,17e,f2daf221,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,5c,...) at mi_switch+0x200
sleepq_switch(c2dedd20,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c2df9210,5c,c0b5db5c,0,0,...) at sleepq_timedwait+0x6b
_sleep(c2df9210,0,5c,c0b5db5c,ea60,...) at _sleep+0x2f9
usb_event_thread(c2df05c0,df3bed38,c0b66eb8,322,c2cc129c,...) at 
usb_event_thread+0xca
fork_exit(c071c160,c2df05c0,df3bed38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3bed70, ebp = 0 ---

Tracing command usb2 pid 18 tid 100029 td 0xc2dee230
sched_switch(c2dee230,0,104,17e,f2dab3b5,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,5c,...) at mi_switch+0x200
sleepq_switch(c2dee230,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c2de0210,5c,c0b5db5c,0,0,...) at sleepq_timedwait+0x6b
_sleep(c2de0210,0,5c,c0b5db5c,ea60,...) at _sleep+0x2f9
usb_event_thread(c2df0d40,df3b7d38,c0b66eb8,322,c2cc1538,...) at 
usb_event_thread+0xca
fork_exit(c071c160,c2df0d40,df3b7d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3b7d70, ebp = 0 ---

Tracing command usb1 pid 17 tid 100027 td 0xc2d07690
sched_switch(c2d07690,0,104,17e,f2d9ecdc,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,5c,...) at mi_switch+0x200
sleepq_switch(c2d07690,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c2de2210,5c,c0b5db5c,0,0,...) at sleepq_timedwait+0x6b
_sleep(c2de2210,0,5c,c0b5db5c,ea60,...) at _sleep+0x2f9
usb_event_thread(c2dc9b80,df3b0d38,c0b66eb8,322,c2cc17d4,...) at 
usb_event_thread+0xca
fork_exit(c071c160,c2dc9b80,df3b0d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3b0d70, ebp = 0 ---

Tracing command usbtask-dr pid 16 tid 100026 td 0xc2d078c0
sched_switch(c2d078c0,0,104,17e,1e986354,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,5c,...) at mi_switch+0x200
sleepq_switch(c2d078c0,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c0cae294,5c,c0b5db4e,0,0,...) at sleepq_wait+0x63
_sleep(c0cae294,0,5c,c0b5db4e,0,...) at _sleep+0x32b
usb_task_thread(c0cae294,df3a9d38,c0b66eb8,322,c2cc1a70,...) at 
usb_task_thread+0x62
fork_exit(c071c020,c0cae294,df3a9d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3a9d70, ebp = 0 ---

Tracing command usbtask-hc pid 15 tid 100025 td 0xc2d07af0
sched_switch(c2d07af0,0,104,17e,1e9847cf,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,5c,...) at mi_switch+0x200
sleepq_switch(c2d07af0,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c0cae280,5c,c0b5db4e,0,0,...) at sleepq_wait+0x63
_sleep(c0cae280,0,5c,c0b5db4e,0,...) at _sleep+0x32b
usb_task_thread(c0cae280,df3a6d38,c0b66eb8,322,c2cc1d0c,...) at 
usb_task_thread+0x62
fork_exit(c071c020,c0cae280,df3a6d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3a6d70, ebp = 0 ---

Tracing command usb0 pid 14 tid 100024 td 0xc2d07d20
sched_switch(c2d07d20,0,104,17e,f2daa198,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,5c,...) at mi_switch+0x200
sleepq_switch(c2d07d20,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c2dd1210,5c,c0b5db5c,0,0,...) at sleepq_timedwait+0x6b
_sleep(c2dd1210,0,5c,c0b5db5c,ea60,...) at _sleep+0x2f9
usb_event_thread(c2dd6a40,df3a3d38,c0b66eb8,322,c2d39000,...) at 
usb_event_thread+0xca
fork_exit(c071c160,c2dd6a40,df3a3d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3a3d70, ebp = 0 ---

Tracing command xpt_thrd pid 5 tid 100018 td 0xc2dc4af0
sched_switch(c2dc4af0,0,104,17e,1e980473,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,4c,...) at mi_switch+0x200
sleepq_switch(c2dc4af0,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c0c7e054,4c,c0b16e25,0,0,...) at sleepq_wait+0x63
_sleep(c0c7e054,c0c7e06c,4c,c0b16e25,0,...) at _sleep+0x32b
xpt_scanner_thread(0,df368d38,c0b66eb8,322,c2d3929c,...) at 
xpt_scanner_thread+0x41
fork_exit(c0471620,0,df368d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf368d70, ebp = 0 ---

Tracing command yarrow pid 13 tid 100013 td 0xc2cc4af0
sched_switch(c2cc4af0,0,104,17e,34db65ae,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,0,...) at mi_switch+0x200
sleepq_switch(c2cc4af0,0,c0b6ec3d,26a,2,...) at sleepq_switch+0x15f
sleepq_timedwait(c0cb0bc4,0,c0b61a9d,2,0,...) at sleepq_timedwait+0x6b
_sleep(c0cb0bc4,0,0,c0b61a9d,64,...) at _sleep+0x2f9
pause(c0b61a9d,64,c0b53550,112,0,...) at pause+0x47
random_kthread(0,c2bfcd38,c0b66eb8,322,c2d39538,...) at random_kthread+0x1d9
fork_exit(c068adc0,0,c2bfcd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2bfcd70, ebp = 0 ---

Tracing command g_down pid 4 tid 100012 td 0xc2cc4d20
sched_switch(c2cc4d20,0,104,17e,34da50b6,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,4c,...) at mi_switch+0x200
sleepq_switch(c2cc4d20,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c0cae9c4,4c,c0b61a9d,0,0,...) at sleepq_timedwait+0x6b
_sleep(c0cae9c4,c0cae928,24c,c0b61a9d,64,...) at _sleep+0x2f9
g_io_schedule_down(c2cc4d20,0,c0b63202,74,0,...) at g_io_schedule_down+0x6b
g_down_procbody(0,c2bf9d38,c0b66eb8,322,c2cc0000,...) at g_down_procbody+0x8d
fork_exit(c07600e0,0,c2bf9d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2bf9d70, ebp = 0 ---

Tracing command g_up pid 3 tid 100011 td 0xc2d07000
sched_switch(c2d07000,0,104,17e,487e35fa,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,4c,...) at mi_switch+0x200
sleepq_switch(c2d07000,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c0cae9c0,4c,c0b61a9d,0,0,...) at sleepq_timedwait+0x6b
_sleep(c0cae9c0,c0cae948,24c,c0b61a9d,64,...) at _sleep+0x2f9
g_io_schedule_up(c2d07000,0,c0b63202,5d,0,...) at g_io_schedule_up+0x133
g_up_procbody(0,c2bf6d38,c0b66eb8,322,c2cc029c,...) at g_up_procbody+0x8d
fork_exit(c0760170,0,c2bf6d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2bf6d70, ebp = 0 ---

Tracing command g_event pid 2 tid 100010 td 0xc2d07230
sched_switch(c2d07230,0,104,17e,4383ceef,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,4c,...) at mi_switch+0x200
sleepq_switch(c2d07230,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c2f05738,4c,c0b6315f,0,0,...) at sleepq_timedwait+0x6b
_sleep(c2f05738,c2c8a7b8,4c,c0b6315f,64,...) at _sleep+0x2f9
biowait(c2f05738,c0b6315f,0,c2bf3b10,c2bf3c10,...) at biowait+0x7d
g_write_data(c2e3ea40,2b476000,1,c2f4b200,200,...) at g_write_data+0xae
g_mirror_write_metadata(c2e47600,c2e9de80,c2bf3b90,2de,4d4f4547,...) at 
g_mirror_write_metadata+0x3c0
g_mirror_update_metadata(c2e9de80,4,c0f8512a,205,c2e4762c,...) at 
g_mirror_update_metadata+0x82
g_mirror_destroy_device(c2e4762c,0,c0f8512a,ba1,c8,...) at 
g_mirror_destroy_device+0x90
g_mirror_destroy(c2e47600,0,c0f8512a,ad9,c2e4762c,...) at 
g_mirror_destroy+0x229
g_mirror_destroy_delayed(c2e47600,0,c0b62d0c,d2,20,...) at 
g_mirror_destroy_delayed+0xa6
g_run_events(c0cae9b8,0,4c,c0b61a9d,64,...) at g_run_events+0x31e
g_event_procbody(0,c2bf3d38,c0b66eb8,322,c2cc0538,...) at 
g_event_procbody+0x8a
fork_exit(c0760200,0,c2bf3d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2bf3d70, ebp = 0 ---

Tracing command intr pid 12 tid 100039 td 0xc2e4d000
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100038 td 0xc2dc4d20
sched_switch(c2dc4d20,0,109,17e,e3949f63,...) at sched_switch+0x437
mi_switch(109,0,c0b67156,4cd,c2e3c7ec,...) at mi_switch+0x200
ithread_loop(c2e46610,df3f5d38,c0b66eb8,322,c2cc07d4,...) at 
ithread_loop+0x34c
fork_exit(c0795810,c2e46610,df3f5d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf3f5d70, ebp = 0 ---

Tracing command intr pid 12 tid 100037 td 0xc2ded000
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100035 td 0xc2ded460
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100034 td 0xc2ded690
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100032 td 0xc2dedaf0
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100030 td 0xc2dee000
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100028 td 0xc2dee460
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100023 td 0xc2dc4000
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100022 td 0xc2dc4230
sched_switch(c2dc4230,0,109,17e,34e83fe0,...) at sched_switch+0x437
mi_switch(109,0,c0b67156,4cd,c2ce676c,...) at mi_switch+0x200
ithread_loop(c2dd2930,df39cd38,c0b66eb8,322,c2cc07d4,...) at 
ithread_loop+0x34c
fork_exit(c0795810,c2dd2930,df39cd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf39cd70, ebp = 0 ---

Tracing command intr pid 12 tid 100021 td 0xc2dc4460
sched_switch(c2dc4460,0,109,17e,e4f9f209,...) at sched_switch+0x437
mi_switch(109,0,c0b67156,4cd,c2ce67ec,...) at mi_switch+0x200
ithread_loop(c2dd2aa0,df395d38,c0b66eb8,322,c2cc07d4,...) at 
ithread_loop+0x34c
fork_exit(c0795810,c2dd2aa0,df395d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf395d70, ebp = 0 ---

Tracing command intr pid 12 tid 100020 td 0xc2dc4690
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100019 td 0xc2dc48c0
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100016 td 0xc2cc4460
sched_switch(c2cc4460,0,109,17e,357cc14c,...) at sched_switch+0x437
mi_switch(109,0,c0b67156,4cd,c2d0f86c,...) at mi_switch+0x200
ithread_loop(c2dc31e0,df362d38,c0b66eb8,322,c2cc07d4,...) at 
ithread_loop+0x34c
fork_exit(c0795810,c2dc31e0,df362d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf362d70, ebp = 0 ---

Tracing command intr pid 12 tid 100015 td 0xc2cc4690
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100008 td 0xc2cc2000
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100007 td 0xc2cc2230
fork_trampoline() at fork_trampoline

Tracing command intr pid 12 tid 100006 td 0xc2cc2460
sched_switch(c2cc2460,0,109,17e,487e1a5b,...) at sched_switch+0x437
mi_switch(109,0,c0b67156,4cd,c2ce62ec,...) at mi_switch+0x200
ithread_loop(c2cbf6b0,c2be7d38,c0b66eb8,322,c2cc07d4,...) at 
ithread_loop+0x34c
fork_exit(c0795810,c2cbf6b0,c2be7d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2be7d70, ebp = 0 ---

Tracing command intr pid 12 tid 100005 td 0xc2cc2690
sched_switch(c2cc2690,0,103,179,353ee944,...) at sched_switch+0x437
mi_switch(103,0,c0b6f4b6,2e0,c2c8b2d0,...) at mi_switch+0x200
turnstile_wait(c2c8b2d0,c2fbe000,0,188,c0cb0630,...) at turnstile_wait+0x48a
_mtx_lock_sleep(c0cb0630,c2cc2690,0,c0b695d3,89,...) at _mtx_lock_sleep+0x18e
_mtx_lock_flags(c0cb0630,0,c0b695d3,89,c2be4cc4,...) at _mtx_lock_flags+0xef
lock_mtx(c0cb0630,1,c0b6bee1,16b,c0cb0df4,...) at lock_mtx+0x29
softclock(c0cb0dc0,0,c0b67156,4d3,c2ce636c,...) at softclock+0x1e5
ithread_loop(c2cbf6c0,c2be4d38,c0b66eb8,322,c2cc07d4,...) at 
ithread_loop+0x1b5
fork_exit(c0795810,c2cbf6c0,c2be4d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2be4d70, ebp = 0 ---

Tracing command idle pid 11 tid 100004 td 0xc2cc28c0
sched_switch(c2cc28c0,0,608,179,34e7a79b,...) at sched_switch+0x437
mi_switch(608,0,c0b6b1be,bc,0,...) at mi_switch+0x200
critical_exit(c2ce525c,c2cc28c0,c2ce525c,c2ce6700,f,...) at critical_exit+0xa8
intr_event_handle(c2ce6700,c2be0c68,1f9,2710,0,...) at intr_event_handle+0xba
intr_execute_handlers(c2ce525c,c2be0c68,c2be0ca8,c0aad244,31,...) at 
intr_execute_handlers+0x4f
lapic_handle_intr(31,c2be0c68) at lapic_handle_intr+0x3f
Xapic_isr1() at Xapic_isr1+0x34
--- interrupt, eip = 0xc0ab6a22, esp = 0xc2be0ca8, ebp = 0xc2be0ca8 ---
cpu_idle_acpi(1,c2be0cf8,c07d4d86,1,c2be0cd8,...) at cpu_idle_acpi+0x22
cpu_idle(1,c2be0cd8,c0b6c6e7,3a3,c2cc28c0,...) at cpu_idle+0x1b
sched_idletd(0,c2be0d38,c0b66eb8,322,c2cc0a70,...) at sched_idletd+0x216
fork_exit(c07d4b70,0,c2be0d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2be0d70, ebp = 0 ---

Tracing command idle pid 11 tid 100003 td 0xc2cc2af0
sched_switch(c2cc2af0,0,108,179,31195894,...) at sched_switch+0x437
mi_switch(108,0,c0b6c6e7,3a1,c2cc2af0,...) at mi_switch+0x200
sched_idletd(0,c2bddd38,c0b66eb8,322,c2cc0a70,...) at sched_idletd+0x178
fork_exit(c07d4b70,0,c2bddd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2bddd70, ebp = 0 ---

Tracing command init pid 1 tid 100002 td 0xc2cc2d20
cpustop_handler(2,c2bd9b88,c0ac8add,c2c70340,c2bd9b0c,...) at 
cpustop_handler+0x32
ipi_nmi_handler(c2c70340,c2bd9b0c,c07f0ef7,c0af18fa,c2cc0d0c,...) at 
ipi_nmi_handler+0x2f
trap(c2bd9b94) at trap+0x2d
calltrap() at calltrap+0x6
--- trap 0x13, eip = 0xc07a5ebf, esp = 0xc2bd9bd4, ebp = 0xc2bd9be8 ---
_mtx_unlock_spin_flags(c2cc0d1c,0,c0b6ab59,8ff,0,...) at 
_mtx_unlock_spin_flags+0xef
tdsigwakeup(4,b,c2bd9ccc,7ea,0,...) at tdsigwakeup+0x212
tdsignal(c2cc0d0c,c2cc2d20,b,c2bd9ccc,a,...) at tdsignal+0x897
trapsignal(c2cc2d20,c2bd9ccc,c0b9ee1a,c2c73330,c2cc0d0c,...) at 
trapsignal+0x297
trap(c2bd9d38) at trap+0x6ad
calltrap() at calltrap+0x6
--- trap 0xc, eip = 0x8048870, esp = 0xbfbfe62c, ebp = 0xbfbfe978 ---

Tracing command audit pid 10 tid 100001 td 0xc2cc4000
sched_switch(c2cc4000,0,104,17e,1e93e7fd,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,0,...) at mi_switch+0x200
sleepq_switch(c2cc4000,0,c0b6ec3d,247,c2cc4000,...) at sleepq_switch+0x15f
sleepq_wait(c0e21ee0,0,c0b8b0fc,1,0,...) at sleepq_wait+0x63
_cv_wait(c0e21ee0,c0e21ec4,c0b8badb,18d,0,...) at _cv_wait+0x213
audit_worker(0,c2bd6d38,c0b66eb8,322,c2cc1000,...) at audit_worker+0x84
fork_exit(c0994b50,0,c2bd6d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2bd6d70, ebp = 0 ---

Tracing command kernel pid 0 tid 100017 td 0xc2cc4230
sched_switch(c2cc4230,0,104,17e,1e9ddb91,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,0,...) at mi_switch+0x200
sleepq_switch(c2cc4230,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c2db1140,0,c0b61a9d,0,0,...) at sleepq_wait+0x63
_sleep(c2db1140,c2db115c,0,c0b61a9d,0,...) at _sleep+0x32b
taskqueue_thread_loop(c0caf1bc,df365d38,c0b66eb8,322,c0caea80,...) at 
taskqueue_thread_loop+0xb4
fork_exit(c07eb240,c0caf1bc,df365d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdf365d70, ebp = 0 ---

Tracing command kernel pid 0 tid 100014 td 0xc2cc48c0
sched_switch(c2cc48c0,0,104,17e,1e9d88a2,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,0,...) at mi_switch+0x200
sleepq_switch(c2cc48c0,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c2db1280,0,c0b61a9d,0,0,...) at sleepq_wait+0x63
_sleep(c2db1280,c2db129c,0,c0b61a9d,0,...) at _sleep+0x32b
taskqueue_thread_loop(c0cbc868,c2bffd38,c0b66eb8,322,c0caea80,...) at 
taskqueue_thread_loop+0xb4
fork_exit(c07eb240,c0cbc868,c2bffd38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2bffd70, ebp = 0 ---

Tracing command kernel pid 0 tid 100009 td 0xc2d07460
sched_switch(c2d07460,0,104,17e,737ce18c,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,0,...) at mi_switch+0x200
sleepq_switch(c2d07460,0,c0b6ec3d,247,0,...) at sleepq_switch+0x15f
sleepq_wait(c2ca2700,0,c0b61a9d,0,0,...) at sleepq_wait+0x63
_sleep(c2ca2700,c2ca271c,0,c0b61a9d,0,...) at _sleep+0x32b
taskqueue_thread_loop(c0cbb4e0,c2bf0d38,c0b66eb8,322,c0caea80,...) at 
taskqueue_thread_loop+0xb4
fork_exit(c07eb240,c0cbb4e0,c2bf0d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc2bf0d70, ebp = 0 ---

Tracing command kernel pid 0 tid 100000 td 0xc0caed20
sched_switch(c0caed20,0,104,17e,34dc1ee6,...) at sched_switch+0x437
mi_switch(104,0,c0b6ec3d,1d2,44,...) at mi_switch+0x200
sleepq_switch(c0caed20,0,c0b6ec3d,26a,0,...) at sleepq_switch+0x15f
sleepq_timedwait(c0caea80,44,c0b6ce40,0,0,...) at sleepq_timedwait+0x6b
_sleep(c0caea80,0,44,c0b6ce40,2710,...) at _sleep+0x2f9
scheduler(0,141ec00,141ec00,141e000,1425000,...) at scheduler+0x23e
mi_startup() at mi_startup+0x96
begin() at begin+0x2c
db>                                                                                     
Received on Fri Sep 05 2008 - 03:36:43 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:39:34 UTC