panic: mtx_lock() of destroyed mutex _at_ route.c

From: Marcel Moolenaar <xcllnt_at_mac.com>
Date: Sun, 23 Sep 2007 11:11:39 -0700
All,

The following panic happened on pluto1.freebsd.org:

login: Sep 20 09:43:20 pluto1 sshd[77174]: fatal: Read from socket  
failed: Connection reset by peer
Sep 20 14:38:20 pluto1 sshd[77835]: fatal: Read from socket failed:  
Connection reset by peer
Sep 20 14:40:10 pluto1 sshd[53908]: error: setsockopt TCP_NODELAY:  
Connection reset by peer
Sep 20 14:40:10 pluto1 last message repeated 2 times
Sep 20 14:43:31 pluto1 sshd[53908]: error: setsockopt TCP_NODELAY:  
Connection reset by peer
Sep 20 14:43:38 pluto1 last message repeated 24 times
Sep 21 01:23:20 pluto1 sshd[79326]: fatal: Read from socket failed:  
Connection reset by peer
Sep 21 03:13:20 pluto1 sshd[79711]: fatal: Read from socket failed:  
Connection reset by peer
Sep 21 05:11:18 pluto1 sshd[53908]: error: setsockopt TCP_NODELAY:  
Connection reset by peer
Sep 21 05:11:23 pluto1 last message repeated 17 times
Sep 22 19:48:21 pluto1 sshd[86444]: fatal: Read from socket failed:  
Connection reset by peer
Sep 22 20:55:15 pluto1 sshd[53908]: error: setsockopt TCP_NODELAY:  
Connection reset by peer
Sep 22 20:55:20 pluto1 last message repeated 8 times
Sep 23 01:28:54 pluto1 sshd[53908]: error: setsockopt TCP_NODELAY:  
Connection reset by peer
Sep 23 01:28:59 pluto1 last message repeated 55 times
panic: mtx_lock() of destroyed mutex _at_ /p/7.x/src/sys/net/route.c:1303
cpuid = 1
KDB: enter: panic
[thread pid 53908 tid 100111 ]
Stopped at      kdb_enter+0x81: [F1]    nop.f 0x0
db> ps
   pid  ppid  pgrp   uid   state   wmesg         wchan        cmd
53909 53908 53909   618  Ss+     ttyin    0xe000000010433010 tcsh
53908   687 53908     0  Rs      CPU 1                       sshd
   782     1   782     0  Ss+     ttyin    0xe00000001052cc10 getty
   781     1   781     0  Ss+     ttyin    0xe000000010536010 getty
   764     1   764     0  Ss      select   0xe00000000480ce30 inetd
   706     1   706     0  Ss      nanslp   0xe000000004778b00 cron
   699     1   699    25  Ss      pause    0xe0000000109576a0 sendmail
   693     1   693     0  Ss      select   0xe00000000480ce30 sendmail
   687     1   687     0  Ss      select   0xe00000000480ce30 sshd
   678     1   678     1  Ss      sbwait   0xe000000010913464 rwhod
   665     1   665     0  Ss      select   0xe00000000480ce30 ntpd
   618   614   614     0  S       -        0xe0000000106f3a00 nfsd
   617   614   614     0  S       -        0xe000000010749600 nfsd
   616   614   614     0  S       -        0xe000000010747a00 nfsd
   615   614   614     0  S       -        0xe0000000106f1600 nfsd
   614     1   614     0  Ss      select   0xe00000000480ce30 nfsd
   612     1   612     0  Ss      select   0xe00000000480ce30 mountd
   555     1   555     0  Ss      select   0xe00000000480ce30 ypbind
   542     1   542     0  Ss      select   0xe00000000480ce30 rpcbind
   518     1   518     0  Ss      select   0xe00000000480ce30 syslogd
   458     1   458     0  Ss      select   0xe00000000480ce30 devd
   159     1   159     0  Ss      pause    0xe0000000106ea520 adjkerntz
    46     0     0     0  SL      -        0xe000000004778a20 [schedcpu]
    45     0     0     0  SL      sdflush  0xe000000004779ac8  
[softdepflush]
    44     0     0     0  SL      syncer   0xe000000004778a24 [syncer]
    43     0     0     0  SL      vlruwt   0xe00000001059cd20 [vnlru]
    42     0     0     0  SL      psleep   0xe0000000047790d4  
[bufdaemon]
    41     0     0     0  SL      pgzero   0xe000000004779da4 [pagezero]
    40     0     0     0  SL      psleep   0xe000000004779d50 [vmdaemon]
    39     0     0     0  SL      psleep   0xe000000004779d64  
[pagedaemon]
    38     0     0     0  WL                                  [swi0:  
uart uart+++]
    37     0     0     0  RL      CPU 0                       [irq29:  
bge0]
    36     0     0     0  SL      idle     0xa000000000022000  
[mpt_recovery1]
    35     0     0     0  WL                                  [irq28:  
mpt1]
    34     0     0     0  SL      idle     0xa00000000000c000  
[mpt_recovery0]
    33     0     0     0  WL                                  [irq27:  
mpt0]
    32     0     0     0  WL                                  [irq20:  
fxp0]
    31     0     0     0  WL                                  [irq21:  
atapci0]
    30     0     0     0  SL      usbevt   0xe000000010469420 [usb2]
    29     0     0     0  WL                                  [irq18:  
ehci0]
    28     0     0     0  SL      usbevt   0xe000000010442420 [usb1]
    27     0     0     0  WL                                  [irq17:  
ohci1]
    26     0     0     0  SL      usbtsk   0xe00000000477fd78  
[usbtask-dr]
    25     0     0     0  SL      usbtsk   0xe00000000477fd50  
[usbtask-hc]
    24     0     0     0  SL      usbevt   0xe0000000103b0420 [usb0]
    23     0     0     0  WL                                  [irq16:  
ohci0]
    22     0     0     0  SL      tzpoll   0xe000000004778148  
[acpi_thermal]
    21     0     0     0  WL                                  [irq36:  
acpi0]
    20     0     0     0  WL                                  [swi6:  
task queue]
    19     0     0     0  WL                                  [swi6:  
Giant taskq]
    18     0     0     0  SL      -        0xe00000001036ee00 [thread  
taskq]
    17     0     0     0  WL                                  [swi5: +]
    16     0     0     0  WL                                  [swi2:  
cambio]
     9     0     0     0  SL      ccb_scan 0xe00000000477a020 [xpt_thrd]
     8     0     0     0  SL      -        0xe00000001036f180 [kqueue  
taskq]
     7     0     0     0  SL      -        0xe00000001036f200  
[acpi_task_2]
     6     0     0     0  SL      -        0xe00000001036f200  
[acpi_task_1]
     5     0     0     0  SL      -        0xe00000001036f200  
[acpi_task_0]
    15     0     0     0  SL      -        0xe000000004778a20 [yarrow]
     4     0     0     0  SL      -        0xe0000000047785b8 [g_down]
     3     0     0     0  SL      -        0xe0000000047785b0 [g_up]
     2     0     0     0  SL      -        0xe0000000047785a0 [g_event]
    14     0     0     0  WL                                  [swi1:  
net]
    13     0     0     0  WL                                  [swi3: vm]
    12     0     0     0  RL                                  [swi4:  
clock]
    11     0     0     0  RL                                  [idle:  
cpu0]
    10     0     0     0  RL                                  [idle:  
cpu1]
     1     0     1     0  SLs     wait     0xe000000010372000 [init]
     0     0     0     0  WLs                                 [swapper]
db> bt
Tracing pid 53908 tid 100111 td 0xe000000010959440
kdb_enter(0xe000000004660b20, 0xe0000000042e2ad0, 0x793,  
0xa000000033b9f1a8) at kdb_enter+0x81
panic(0xe00000000465f758, 0xe000000004672a18, 0x517, 0x507,  
0xe00000000480ccf8) at panic+0x2e0
_mtx_lock_flags(0xe00000002ae49920, 0x0, 0xe000000004672a18, 0x517)  
at _mtx_lock_flags+0x90
rt_check(0xa000000033b9f1e0, 0xa000000033b9f1f0, 0xe00000002ae49920,  
0xe00000002ae49860) at rt_check+0x1e0
arpresolve(0xe000000010351000, 0xe00000001073b770,  
0xe000000010754e00, 0xe000000010542ef0, 0xa000000033b9f208) at  
arpresolve+0x170
ether_output(0xe000000010351000, 0xe000000010754e00,  
0xe000000010542ef0, 0xe00000001073b770, 0x0) at ether_output+0x120
ip_output(0xe000000010754e62, 0xe000000010542ef0, 0xa000000033b9f228,  
0x0, 0x0, 0xe0000000108f0230, 0xe000000010754e6a) at ip_output+0x19e0
tcp_output(0xe0000000109d10e0, 0xe0000000109d1140,  
0xe0000000109d11e8, 0xe00000000480fc20) at tcp_output+0x2ab0
tcp_usr_shutdown(0xe00000407bbd82b8) at tcp_usr_shutdown+0x100
soshutdown(0xe00000407bbd82b8, 0x1, 0xe00000000474bf10,  
0xe00000000439d2f0) at soshutdown+0xb0
shutdown(0xe000000010959440, 0xa000000033b9f4e8, 0x0,  
0xe00000000461e3e0) at shutdown+0x90
syscall(0xa000000033b9f400, 0x86, 0x20000000401331d0,  
0xe000000010959440, 0xe00000001095fa40, 0xe000000004734948, 0x86,  
0xa000000033b9f4e8) at syscall+0x3e0
epc_syscall_return() at epc_syscall_return
db> thread 100028
[thread pid 37 tid 100028 ]
interrupt+0x760:        [M0]    ld4.acq r14=[r36]
db> bt
Tracing pid 37 tid 100028 td 0xe0000000103757a0
interrupt(0xa000000031349000) at interrupt+0x760
ivt_External_Interrupt() at ivt_External_Interrupt+0x30
--- trapframe at 0xa000000031349000
_mtx_lock_sleep(0xe00000001073b830, 0xe0000000103757a0, 0x0,  
0xe00000001073b850, 0xe000000010959440) at _mtx_lock_sleep+0x221
_mtx_lock_flags(0xe00000001073b830, 0x0, 0xe000000004672a18, 0x524)  
at _mtx_lock_flags+0x170
rt_check(0xa000000031349260, 0xa000000031349270, 0xe000000004672a18,  
0xe0000000120d2000) at rt_check+0x420
arpresolve(0xe000000010351000, 0xe00000001073b770,  
0xe000000012faf700, 0xe000000010542ef0, 0xa000000031349288) at  
arpresolve+0x170
ether_output(0xe000000010351000, 0xe000000012faf700,  
0xe000000010542ef0, 0xe00000001073b770, 0x0) at ether_output+0x120
ip_output(0xe000000012faf762, 0xe000000010542ef0, 0xa0000000313492a8,  
0x0, 0x0, 0xe0000000143f8c08, 0xe000000012faf76a) at ip_output+0x19e0
tcp_output(0xe0000000108f42d0, 0xe0000000108f4330,  
0xe0000000108f43d8, 0xe00000000480fc20, 0xe00000000480fbf8) at  
tcp_output+0x2ab0
tcp_do_segment(0xe000004078d50e00, 0xe000004048d35024,  
0xe000000010913b30, 0xe0000000108f42d0, 0x34, 0x18, 0x0) at  
tcp_do_segment+0x1560
tcp_input(0xe000004078d50e00, 0x34, 0xe000004078d50e28,  
0xe000004048d35026, 0xe0000000143f8c08) at tcp_input+0x1b30
ip_input(0xe000004048d35010, 0x14, 0x0, 0x14) at ip_input+0x13f0
netisr_dispatch(0x2, 0xe000004078d50e00, 0xe00000000440db60, 0x289)  
at netisr_dispatch+0x190
ether_demux(0xa000000031349530, 0x800, 0xe00000000440e870, 0x30b,  
0xe000000004960060) at ether_demux+0x570
ether_input(0xe000000010351000, 0xe000004048d35002,  
0x800000000000000, 0xe00000000414e880) at ether_input+0xa50
bge_intr(0xe000004078d50e00, 0xe000004078d50e10, 0xa0000000314e00e0,  
0xa000000000038000) at bge_intr+0x1340
ithread_loop(0xe00000001049a8c0, 0xe000000010520a00, 0x0,  
0xe00000000465c778) at ithread_loop+0x420
fork_exit(0xe000000004695dc0, 0xe00000001049a8c0, 0xa000000031349550)  
at fork_exit+0x110
enter_userland() at enter_userland
db> show alllocks
Process 53908 (sshd) thread 0xe000000010959440 (100111)
exclusive sleep mutex rtentry r = 0 (0xe00000001073b830) locked _at_ /p/ 
7.x/src/sys/net/route.c:1287
exclusive sleep mutex inp (tcpinp) r = 0 (0xe0000000108f0318) locked  
_at_ /p/7.x/src/sys/netinet/tcp_usrreq.c:695
exclusive sleep mutex tcp r = 0 (0xe00000000480fb40) locked _at_ /p/7.x/ 
src/sys/netinet/tcp_usrreq.c:692
Process 678 (rwhod) thread 0xe0000000106ecd80 (100076)
exclusive sx so_rcv_sx r = 0 (0xe000000010913408) locked _at_ /p/7.x/src/ 
sys/kern/uipc_sockbuf.c:145
Process 37 (irq29: bge0) thread 0xe0000000103757a0 (100028)
exclusive sleep mutex rtentry r = 0 (0xe0000000120d20c0) locked _at_ /p/ 
7.x/src/sys/net/route.c:788
exclusive sleep mutex radix node head r = 1 (0xe0000000105520f0)  
locked _at_ /p/7.x/src/sys/net/route.c:684
exclusive sleep mutex inp (tcpinp) r = 0 (0xe0000000143f8cf0) locked  
_at_ /p/7.x/src/sys/netinet/tcp_input.c:477
db>

FYI,

-- 
Marcel Moolenaar
xcllnt_at_mac.com
Received on Sun Sep 23 2007 - 16:13:03 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:39:18 UTC