Re: panic: mtx_lock() of destroyed mutex _at_ /usr/src/sys/net/route.c:1306

On Mar 27, 2007, at 1:23 AM, Andre Guibert de Bruet wrote:

> I got this earlier today. I managed to get what appears to be a  
> sane dump:
>
> Unread portion of the kernel message buffer:
> panic: mtx_lock() of destroyed mutex _at_ /usr/src/sys/net/route.c:1306
> cpuid = 0
> KDB: enter: panic
> Physical memory: 3575 MB
> Dumping 324 MB: 309 293 277 261 245 229 213 197 181 165 149 133 117  
> 101 85 69 53 37 21 5
>
> #0  doadump () at pcpu.h:172
> 172     pcpu.h: No such file or directory.
>         in pcpu.h
> (kgdb) bt
> #0  doadump () at pcpu.h:172
> #1  0xc0451f5d in db_fncall (dummy1=0, dummy2=0, dummy3=2399,  
> dummy4=0xe545b704 "") at /usr/src/sys/ddb/db_command.c:486
> #2  0xc0451d2c in db_command (last_cmdp=0xc07ae024, cmd_table=0x0)  
> at /usr/src/sys/ddb/db_command.c:401
> #3  0xc0451df3 in db_command_loop () at /usr/src/sys/ddb/ 
> db_command.c:453
> #4  0xc0453d61 in db_trap (type=3, code=0) at /usr/src/sys/ddb/ 
> db_main.c:222
> #5  0xc0592d9f in kdb_trap (type=0, code=0, tf=0xe545b8a8) at /usr/ 
> src/sys/kern/subr_kdb.c:502
> #6  0xc070007b in trap (frame=0xe545b8a8) at /usr/src/sys/i386/i386/ 
> trap.c:621
> #7  0xc06e729b in calltrap () at /usr/src/sys/i386/i386/exception.s: 
> 139
> #8  0xc0592ac0 in kdb_enter (msg=0x12 <Address 0x12 out of bounds>)  
> at cpufunc.h:60
> #9  0xc056e0e5 in panic (fmt=0xc074451f "mtx_lock() of destroyed  
> mutex _at_ %s:%d") at /usr/src/sys/kern/kern_shutdown.c:547
> #10 0xc0563397 in _mtx_lock_flags (m=0xc70322b8, opts=0,  
> file=0xc075261b "/usr/src/sys/net/route.c", line=1306) at /usr/src/ 
> sys/kern/kern_mutex.c:186
> #11 0xc060b768 in rt_check (lrt=0x12, lrt0=0xe545b9a0, dst=0x12)  
> at /usr/src/sys/net/route.c:1306
> #12 0xc060ec84 in arpresolve (ifp=0xc6a3fc00, rt0=0xc6f8ee88,  
> m=0xc6c6e700, dst=0xc6ac88b0, desten=0xe545b9c0 "?E??r`?\0309 
> \201?") at /usr/src/sys/netinet/if_ether.c:378
> #13 0xc05fdcee in ether_output (ifp=0xc6a3fc00, m=0xc6c6e700,  
> dst=0xc6ac88b0, rt0=0x12) at /usr/src/sys/net/if_ethersubr.c:170
> #14 0xc0621f6e in ip_output (m=0xc6c6e700, opt=0x1, ro=0xe545ba28,  
> flags=0, imo=0x0, inp=0xc71c2000) at /usr/src/sys/netinet/ 
> ip_output.c:561
> #15 0xc062af48 in tcp_output (tp=0xc71ce910) at /usr/src/sys/ 
> netinet/tcp_output.c:1122
> #16 0xc06290bc in tcp_do_segment (m=0xc6c6e700, th=0xc6c6e758,  
> so=0xca1282b8, tp=0xc71ce910, drop_hdrlen=40, tlen=0) at /usr/src/ 
> sys/netinet/tcp_input.c:2537
> #17 0xc0627041 in tcp_input (m=0xc6c6e700, off0=40) at /usr/src/sys/ 
> netinet/tcp_input.c:1004
> #18 0xc061f5c4 in ip_input (m=0xc6c6e700) at /usr/src/sys/netinet/ 
> ip_input.c:662
> #19 0xc0605c2a in netisr_processqueue (ni=0xc0812dd8) at /usr/src/ 
> sys/net/netisr.c:236
> #20 0xc0605e64 in swi_net (dummy=0x0) at /usr/src/sys/net/netisr.c:349
> #21 0xc055514a in ithread_execute_handlers (p=0xc6947900,  
> ie=0xc6988580) at /usr/src/sys/kern/kern_intr.c:682
> #22 0xc055528f in ithread_loop (arg=0xc6887940) at /usr/src/sys/ 
> kern/kern_intr.c:766
> #23 0xc0553e4c in fork_exit (callout=0xc0555217 <ithread_loop>,  
> arg=0x12, frame=0x12) at /usr/src/sys/kern/kern_fork.c:814
> #24 0xc06e7310 in fork_trampoline () at /usr/src/sys/i386/i386/ 
> exception.s:205

(kgdb) frame 11
#11 0xc060b768 in rt_check (lrt=0x12, lrt0=0xe545b9a0, dst=0x12) at / 
usr/src/sys/net/route.c:1306
1306                    RT_LOCK(rt);            /* NB: gwroute */
(kgdb) list
1301            /* XXX BSD/OS checks dst->sa_family != AF_NS */
1302            if (rt->rt_flags & RTF_GATEWAY) {
1303                    if (rt->rt_gwroute == NULL)
1304                            goto lookup;
1305                    rt = rt->rt_gwroute;
1306                    RT_LOCK(rt);            /* NB: gwroute */
1307                    if ((rt->rt_flags & RTF_UP) == 0) {
1308                            rtfree(rt);     /* unlock gwroute */
1309                            rt = rt0;
1310                    lookup:
(kgdb) inspect *rt
$12 = {rt_nodes = {{rn_mklist = 0x0, rn_parent = 0xc7032270, rn_bit =  
-1, rn_bmask = 0 '\0', rn_flags = 0 '\0', rn_u = {rn_leaf = {
           rn_Key = 0xc9435400  
"??????????????????????????????????????????????????????????????????????? 
?????????????????????????????????????????????????????? 
My?????????????????????????????????????????????????????????????????????? 
???"..., rn_Mask = 0x0, rn_Dupedkey = 0x0}, rn_node = {rn_Off =  
-918334464, rn_L = 0x0, rn_R = 0x0}}}, {
       rn_mklist = 0x0, rn_parent = 0xc7032090, rn_bit = 63, rn_bmask  
= 1 '\001', rn_flags = 0 '\0', rn_u = {rn_leaf = {rn_Key = 0x7  
<Address 0x7 out of bounds>,
           rn_Mask = 0xc70320f0 "\020????$\003???", rn_Dupedkey =  
0xc7032258}, rn_node = {rn_Off = 7, rn_L = 0xc70320f0, rn_R =  
0xc7032258}}}}, rt_gateway = 0xc9435410,
   rt_flags = 131076, rt_ifp = 0xc6a3fc00, rt_ifa = 0xc6cbf200,  
rt_rmx = {rmx_mtu = 1500, rmx_expire = 106852, rmx_pksent = 0},  
rt_refcnt = 0, rt_genmask = 0x0,
   rt_llinfo = 0x0, rt_gwroute = 0x0, rt_parent = 0x0, rt_mtx =  
{lock_object = {lo_name = 0xc07498ec "rtentry", lo_type = 0xc07498ec  
"rtentry", lo_flags = 21102592,
       lo_witness_data = {lod_list = {stqe_next = 0xc07d5050},  
lod_witness = 0xc07d5050}}, mtx_lock = 6, mtx_recurse = 0}}
(kgdb) inspect **lrt0
$42 = {rt_nodes = {{rn_mklist = 0xc6c7f7a0, rn_parent = 0xc6ac1e34,  
rn_bit = -1, rn_bmask = 0 '\0', rn_flags = 4 '\004', rn_u = {rn_leaf = {
           rn_Key = 0xc6ac88a0 "\020\002", rn_Mask = 0xc6aaee00 "",  
rn_Dupedkey = 0x0}, rn_node = {rn_Off = -961771360, rn_L =  
0xc6aaee00, rn_R = 0x0}}}, {
       rn_mklist = 0x0, rn_parent = 0x0, rn_bit = 0, rn_bmask = 0  
'\0', rn_flags = 0 '\0', rn_u = {rn_leaf = {rn_Key = 0x0, rn_Mask =  
0x0, rn_Dupedkey = 0x0}, rn_node = {
           rn_Off = 0, rn_L = 0x0, rn_R = 0x0}}}}, rt_gateway =  
0xc6ac88b0, rt_flags = 2051, rt_ifp = 0xc6a3fc00, rt_ifa =  
0xc6cbf200, rt_rmx = {rmx_mtu = 1500,
     rmx_expire = 0, rmx_pksent = 506624}, rt_refcnt = 2, rt_genmask  
= 0x0, rt_llinfo = 0x0, rt_gwroute = 0xc7032258, rt_parent = 0x0,  
rt_mtx = {lock_object = {
       lo_name = 0xc07498ec "rtentry", lo_type = 0xc07498ec  
"rtentry", lo_flags = 21168128, lo_witness_data = {lod_list =  
{stqe_next = 0xc07d5050},
         lod_witness = 0xc07d5050}}, mtx_lock = 3331623026,  
mtx_recurse = 0}}

This system's uname -a:
FreeBSD bling.properkernel.com 7.0-CURRENT FreeBSD 7.0-CURRENT #1:  
Sun Mar 25 18:12:26 EDT 2007     andy_at_bling.properkernel.com:/usr/obj/ 
usr/src/sys/BLING  i386

Before I called doadump(), there were a series of hardware watchdog  
timeouts on this system's msk0 interface. I do not know if this is  
related to the problem at hand...

Meanwhile, please excuse my kgdb-fu, as I am still fairly green. If  
there is other output that would help diagnose the problem, please  
let me know.

Cheers,
Andy

/*  Andre Guibert de Bruet  * 6f43 6564 7020 656f 2e74 4220 7469 6a20 */
/*   Code poet / Sysadmin   * 636f 656b 2e79 5320 7379 6461 696d 2e6e */
/*   GSM: +1 734 846 8758   * 5520 494e 2058 6c73 7565 6874 002e 0000 */
/* WWW: siliconlandmark.com * C/C++, Java, Perl, PHP, SQL, XHTML, XML */