Re: panic: thread's turnstile has pending threads

From: Mark Atkinson <atkin901_at_yahoo.com>
Date: Thu, 19 Apr 2007 13:17:11 -0700
Mark Atkinson wrote:

> 7.0-CURRENT FreeBSD 7.0-CURRENT #6: Fri Mar 30 07:14:23 PDT 2007
> 
> Has anyone seen this?   This machine has dual CPUs plus two shared
> interrupt 8 port serial I/O cards under uart/puc.

Additional info:

# kgdb ./kernel.debug /var/crash/vmcore.7
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so:
Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain
conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".

Unread portion of the kernel message buffer:
panic: thread's turnstile has pending threads
cpuid = 1
KDB: enter: panic
exclusive sleep mutex process lock r = 0 (0xc44104e0) locked
_at_ /usr/src/sys/kern/kern_fork.c:417
exclusive sleep mutex process lock r = 0 (0xc72442a0) locked
_at_ /usr/src/sys/kern/kern_fork.c:416
exclusive sleep mutex Giant r = 0 (0xc0b4fd08) locked
_at_ /usr/src/sys/kern/kern_timeout.c:240
exclusive sleep mutex process lock r = 0 (0xc44104e0) locked
_at_ /usr/src/sys/kern/kern_fork.c:417
exclusive sleep mutex process lock r = 0 (0xc72442a0) locked
_at_ /usr/src/sys/kern/kern_fork.c:416
exclusive sleep mutex Giant r = 0 (0xc0b4fd08) locked
_at_ /usr/src/sys/kern/kern_timeout.c:240
panic: from debugger
cpuid = 1
KDB: stack backtrace:
Physical memory: 1011 MB
Dumping 173 MB: 158 142 126 110 94 78 62 46 30 14

#0  doadump () at pcpu.h:172
172             __asm __volatile("movl %%fs:0,%0" : "=r" (td));
(kgdb) where
#0  doadump () at pcpu.h:172
#1  0xc0484cfa in db_fncall (dummy1=1016, dummy2=0, dummy3=1016,
    dummy4=0xe44309b0 "F") at /usr/src/sys/ddb/db_command.c:486
#2  0xc0484aa0 in db_command (last_cmdp=0xc0b24464, cmd_table=0x0)
    at /usr/src/sys/ddb/db_command.c:401
#3  0xc0484b85 in db_command_loop () at /usr/src/sys/ddb/db_command.c:453
#4  0xc0486de5 in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_main.c:222
#5  0xc077bbbf in kdb_trap (type=0, code=0, tf=0xe4430b54)
    at /usr/src/sys/kern/subr_kdb.c:502
#6  0xc09ba7f5 in trap (frame=0xe4430b54)
at /usr/src/sys/i386/i386/trap.c:621
#7  0xc09a011b in calltrap () at /usr/src/sys/i386/i386/exception.s:139
#8  0xc077b850 in kdb_enter (msg=0x12 <Address 0x12 out of bounds>)
    at cpufunc.h:60
#9  0xc0753fbb in panic (
    fmt=0xc0a4f1b1 "thread's turnstile has pending threads")
    at /usr/src/sys/kern/kern_shutdown.c:547
#10 0xc0786ef8 in turnstile_wait (lock=0xc0b4f810, owner=0xc4411d80,
queue=0)
    at /usr/src/sys/kern/subr_turnstile.c:616
#11 0xc0748e6a in _mtx_lock_sleep (m=0xc0b4f810, tid=3288649728, opts=0,
    file=0x0, line=0) at /usr/src/sys/kern/kern_mutex.c:411
#12 0xc074881f in _mtx_lock_flags (m=0xc0b4f810, opts=0,
    file=0xc0a4a388 "/usr/src/sys/kern/kern_mutex.c", line=149)
    at /usr/src/sys/kern/kern_mutex.c:194
#13 0xc0748689 in lock_mtx (lock=0x12, how=0)
    at /usr/src/sys/kern/kern_mutex.c:149
#14 0xc07205d3 in _cv_wait (cvp=0xc0b4fdf8, lock=0xc0b4f810)
    at /usr/src/sys/kern/kern_condvar.c:137
#15 0xc075b122 in _sx_slock (sx=0xc0b4fde0,
    file=0xc0a4cdd2 "/usr/src/sys/kern/sched_4bsd.c", line=378)
    at /usr/src/sys/kern/kern_sx.c:160
#16 0xc076ed05 in schedcpu () at /usr/src/sys/kern/sched_4bsd.c:378
#17 0xc076eef5 in schedcpu_thread () at /usr/src/sys/kern/sched_4bsd.c:478
#18 0xc0738241 in fork_exit (callout=0xc076eee0 <schedcpu_thread>, arg=0x12,
    frame=0x12) at /usr/src/sys/kern/kern_fork.c:814
#19 0xc09a0190 in fork_trampoline ()
at /usr/src/sys/i386/i386/exception.s:205
(kgdb) alltrace
Undefined command: "alltrace".  Try "help".
(kgdb) frame 10
#10 0xc0786ef8 in turnstile_wait (lock=0xc0b4f810, owner=0xc4411d80,
queue=0)
    at /usr/src/sys/kern/subr_turnstile.c:616
616                     KASSERT(TAILQ_EMPTY(&ts->ts_pending),
(kgdb) list
611                                     turnstile_max_depth =
tc->tc_max_depth;
612                     }
613     #endif
614                     ts = td->td_turnstile;
615                     LIST_INSERT_HEAD(&tc->tc_turnstiles, ts, ts_hash);
616                     KASSERT(TAILQ_EMPTY(&ts->ts_pending),
617                         ("thread's turnstile has pending threads"));
618                    
KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]),
619                         ("thread's turnstile has exclusive waiters"));
620                    
KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]),
(kgdb) p ts
$1 = (struct turnstile *) 0xc404a9c0
(kgdb) p *ts
$2 = {ts_blocked = {{tqh_first = 0x0, tqh_last = 0xc404a9c0}, {
      tqh_first = 0x0, tqh_last = 0xc404a9c8}}, ts_pending = {
    tqh_first = 0x5b, tqh_last = 0xc404a9d0}, ts_hash = {le_next = 0x0,
    le_prev = 0xc0b56860}, ts_link = {le_next = 0x0, le_prev = 0x0},
  ts_free = {lh_first = 0x0}, ts_lockobj = 0x0, ts_owner = 0x0}
(kgdb) frame 11
#11 0xc0748e6a in _mtx_lock_sleep (m=0xc0b4f810, tid=3288649728, opts=0,
    file=0x0, line=0) at /usr/src/sys/kern/kern_mutex.c:411
411                     turnstile_wait(&m->lock_object, mtx_owner(m),
(kgdb) list
406     #endif
407
408                     /*
409                      * Block on the turnstile.
410                      */
411                     turnstile_wait(&m->lock_object, mtx_owner(m),
412                         TS_EXCLUSIVE_QUEUE);
413             }
414     #ifdef KTR
415             if (cont_logged) {
(kgdb) p *m
$3 = {lock_object = {lo_name = 0xc0a4a33f "lockbuilder mtxpool",
    lo_type = 0xc0a4a33f "lockbuilder mtxpool", lo_flags = 17104896,
    lo_witness_data = {lod_list = {stqe_next = 0x0}, lod_witness = 0x0}},
  mtx_lock = 3292601730, mtx_recurse = 0}
(kgdb) frame 12
#12 0xc074881f in _mtx_lock_flags (m=0xc0b4f810, opts=0,
    file=0xc0a4a388 "/usr/src/sys/kern/kern_mutex.c", line=149)
    at /usr/src/sys/kern/kern_mutex.c:194
194             _get_sleep_lock(m, curthread, opts, file, line);
(kgdb) list
189                 ("mtx_lock() of spin mutex %s _at_ %s:%d",
m->lock_object.lo_name,
190                 file, line));
191             WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER |
LOP_EXCLUSIVE,
192                 file, line);
193
194             _get_sleep_lock(m, curthread, opts, file, line);
195             LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse,
file,
196                 line);
197             WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file,
line);
198             curthread->td_locks++;
(kgdb) frame 13
#13 0xc0748689 in lock_mtx (lock=0x12, how=0)
    at /usr/src/sys/kern/kern_mutex.c:149
149             mtx_lock((struct mtx *)lock);
(kgdb) list
144
145     void
146     lock_mtx(struct lock_object *lock, int how)
147     {
148
149             mtx_lock((struct mtx *)lock);
150     }
151
152     void
153     lock_spin(struct lock_object *lock, int how)
(kgdb) frame 14
#14 0xc07205d3 in _cv_wait (cvp=0xc0b4fdf8, lock=0xc0b4f810)
    at /usr/src/sys/kern/kern_condvar.c:137
137             class->lc_lock(lock, lock_state);
(kgdb) list
132     #ifdef KTRACE
133             if (KTRPOINT(td, KTR_CSW))
134                     ktrcsw(0, 0);
135     #endif
136             PICKUP_GIANT();
137             class->lc_lock(lock, lock_state);
138             WITNESS_RESTORE(lock, lock_witness);
139     }
140
141     /*
(kgdb) frame 15
#15 0xc075b122 in _sx_slock (sx=0xc0b4fde0,
    file=0xc0a4cdd2 "/usr/src/sys/kern/sched_4bsd.c", line=378)
    at /usr/src/sys/kern/kern_sx.c:160
160                     cv_wait(&sx->sx_shrd_cv, sx->sx_lock);
(kgdb) list
155              * Loop in case we lose the race for lock acquisition.
156              */
157             while (sx->sx_cnt < 0) {
158                     sx->sx_shrd_wcnt++;
159                     lock_profile_obtain_lock_failed(&sx->lock_object,
&contested, &waittime);
160                     cv_wait(&sx->sx_shrd_cv, sx->sx_lock);
161                     sx->sx_shrd_wcnt--;
162             }
163
164             /* Acquire a shared lock. */
(kgdb)  

-- 
Mark Atkinson
atkin901_at_yahoo.com
(!wired)?(coffee++):(wired);
Received on Thu Apr 19 2007 - 18:17:35 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:39:08 UTC