panic: Assertion td->td_turnstile != NULL failed at ../../../kern/subr_turnstile.c:478

Was doing some stress testing on a box with some if_xl locking tweaks from
Bruce Simpson, and ran into this.  Configuration is a GENERIC kernel
running on a dual-Xeon 2.4GHz (each one with two logical processors, so 
4 HTT processors).  The workload was as follows:

- Apache 1.3 running on the box, 20 recursive wget clients from an
  adjacent machine.

- Simultaneous "supersmack" select key mysql test running with 11 threads
  and 10,000 transactions, running with libkse.

I attempted to log in on the serial console and got this panic:

login: root
Password:panic: Assertion td->td_turnstile != NULL failed at
../../../kern/subr_turnstile.c:478
cpuid = 1; 
Stack backtrace:
backtrace(100,c2ba4840,c088f4a0,c2ba4840,c24ade80) at backtrace+0x12
panic(c07ccf34,c07d2fd8,c07d2dc0,1de,c089a9c0) at panic+0x11e
turnstile_wait(c24ade80,c088f4a0,c22d6b00,c088f4a0,2,c07cf595,20d) at
turnstile_wait+0x23a
_mtx_lock_sleep(c088f4a0,0,c07d873a,208) at _mtx_lock_sleep+0x103
_mtx_lock_flags(c088f4a0,0,c07d873a,208) at _mtx_lock_flags+0x80
vn_read(c291c330,e6ff8c88,c2a54400,1,c2ba4840) at vn_read+0x22
dofileread(c2ba4840,c291c330,20,ac3e528,3b) at dofileread+0xb0
pread(c2ba4840,e6ff8d14,6,a3,202) at pread+0x6d
syscall(ac0002f,ac1002f,bf82002f,3b,0) at syscall+0x217
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (198), eip = 0x28472477, esp = 0xbf82a360, ebp = 0xbf82a38c ---
Debugger("panic")
Stopped at      Debugger+0x46:  xchgl   %ebx,in_Debugger.0
db> show pcpu
cpuid        = 1
curthread    = 0xc2ba4840: pid 601 "mysqld"
curpcb       = 0xe6ff8da0
fpcurthread  = none
idlethread   = 0xc225f840: pid 13 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x30
spin locks held:

---

(kgdb) bt
#0  Debugger (msg=0xc07d01bb "panic") at
../../../i386/i386/db_interface.c:319
#1  0xc05eeaa9 in panic (fmt=0xc07ccf34 "Assertion %s failed at %s:%d")
    at ../../../kern/kern_shutdown.c:543
#2  0xc060bfd6 in turnstile_wait (ts=0xc24ade80, lock=0xc088f4a0, 
    owner=0xc22d6b00) at ../../../kern/subr_turnstile.c:518
#3  0xc05e715b in _mtx_lock_sleep (m=0xc088f4a0, opts=0, 
    file=0xc07d873a "../../../kern/vfs_vnops.c", line=520)
    at ../../../kern/kern_mutex.c:541
#4  0xc05e6db4 in _mtx_lock_flags (m=0xc088f4a0, opts=0, 
    file=0xc07d873a "../../../kern/vfs_vnops.c", line=520)
    at ../../../kern/kern_mutex.c:252
#5  0xc064516a in vn_read (fp=0xc291c330, uio=0xe6ff8c88, 
    active_cred=0xc2a54400, flags=1, td=0xc2ba4840)
    at ../../../kern/vfs_vnops.c:520
#6  0xc060ea70 in dofileread (td=0xc2ba4840, fp=0xc291c330, fd=32, 
    buf=0xac3e528, nbyte=59, offset=-4539276580668571630, flags=1)
    at ../../../sys/file.h:234
#7  0xc060e9a5 in pread (td=0xc2ba4840, uap=0xe6ff8d14)
    at ../../../kern/sys_generic.c:144
#8  0xc0769513 in syscall (frame=
      {tf_fs = 180355119, tf_es = 180420655, tf_ds = -1081999313, tf_edi =
59, tf_esi = 0, tf_ebp = -1081957492, tf_isp = -419459724, tf_ebx =
676218668, tf_edx = 180380160, tf_ecx = 179573384, tf_eax = 198, tf_trapno
= 0, tf_err = 2, tf_eip = 675751031, tf_cs = 31, tf_eflags = 514, tf_esp =
-1081957536, tf_ss = 47})
    at ../../../i386/i386/trap.c:1004
(kgdb) up
#1  0xc05eeaa9 in panic (fmt=0xc07ccf34 "Assertion %s failed at %s:%d")
    at ../../../kern/kern_shutdown.c:543
543                     Debugger ("panic");
(kgdb) up
#2  0xc060bfd6 in turnstile_wait (ts=0xc24ade80, lock=0xc088f4a0, 
    owner=0xc22d6b00) at ../../../kern/subr_turnstile.c:518
518                     MPASS(owner == ts->ts_owner);
(kgdb) inspect *ts
$1 = {ts_blocked = {tqh_first = 0xc2ba49a0, tqh_last = 0xc2ba4440}, 
  ts_pending = {tqh_first = 0x0, tqh_last = 0xc24ade88}, ts_hash = {
    le_next = 0x0, le_prev = 0xc089a9c0}, ts_link = {le_next = 0x0, 
    le_prev = 0xc22d6b74}, ts_free = {lh_first = 0xc2a7d9c0}, 
  ts_lockobj = 0xc088f4a0, ts_owner = 0xc22d6b00}
(kgdb) 
(kgdb) up
#3  0xc05e715b in _mtx_lock_sleep (m=0xc088f4a0, opts=0, 
    file=0xc07d873a "../../../kern/vfs_vnops.c", line=520)
    at ../../../kern/kern_mutex.c:541
541                     turnstile_wait(ts, &m->mtx_object, mtx_owner(m));
(kgdb) inspect *m
$2 = {mtx_object = {lo_class = 0xc082af3c, lo_name = 0xc07e0295 "Giant", 
    lo_type = 0xc07e0295 "Giant", lo_flags = 720896, lo_list = {
      tqe_next = 0xc088f460, tqe_prev = 0xc0899770}, lo_witness =
0xc089cac8}, 
  mtx_lock = 3257756418, mtx_recurse = 0}

Robert N M Watson             FreeBSD Core Team, TrustedBSD Projects
robert_at_fledge.watson.org      Principal Research Scientist, McAfee Research