ktr/alq/witness failure.

From: Julian Elischer <julian_at_elischer.org>
Date: Sun, 26 Sep 2004 23:15:05 -0700
I'm not going to stop and debug this now because I'm chasing another
problem but I can probably stash a corefile somewhere if people want it..


ref4# sysctl debug.ktr.mask=0x2fffffff
debug.ktr.mask: 262144
  ->
805306367
ref4#
ref4#
ref4# sysctl debug.ktr.mask=0x0
kernel trap 12 with interrupts disabled

Fatal double fault:
eip = 0xc053336e
esp = 0xcbaa3ff4
ebp = 0xcbaa401c
cpuid = 0; apic id = 00
panic: double fault
cpuid = 0
KDB: enter: panic
[thread 100027]
Stopped at      kdb_enter+0x2b: nop
db> tr
kdb_enter(c069adef) at kdb_enter+0x2b
panic(c06b581a,c06b59c4,0,0,0) at panic+0x127
dblfault_handler() at dblfault_handler+0x7a
--- trap 0x17, eip = 0xc053336e, esp = 0xcbaa3ff4, ebp = 0xcbaa401c ---
witness_checkorder(c111776c,9,c06b5972,2bb) at witness_checkorder+0x6a
_mtx_lock_flags(c111776c,0,c06b5972,2bb,c10e3a80) at _mtx_lock_flags+0x5b
trap_pfault(cbaa40ac,0,608) at trap_pfault+0xb4
trap(18,5f660010,63730010,9,c111776c) at trap+0x335
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc0533495, esp = 0xcbaa40ec, ebp = 0xcbaa4114 ---
witness_checkorder(c111776c,9,c06b597b,2bb) at witness_checkorder+0x191
_mtx_lock_flags(c111776c,0,c06b5972,2bb,c10e3a80) at _mtx_lock_flags+0x5b
trap_pfault(cbaa41a4,0,608) at trap_pfault+0xb4
trap(9700018,705f0010,6b2e0010,9,c111776c) at trap+0x335
----
etc. (LOTS OF IDENTICAL FRAMES REMOVED)
----
--- trap 0xc, eip = 0xc0533495, esp = 0xcbaa5734, ebp = 0xcbaa575c ---
witness_checkorder(c111776c,9,c06b597b,2bb) at witness_checkorder+0x191
_mtx_lock_flags(c111776c,0,c06b5972,2bb,c10e3a80) at _mtx_lock_flags+0x5b
trap_pfault(cbaa57ec,0,608) at trap_pfault+0xb4
trap(18,10,c0710010,9,c111776c) at trap+0x335
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc0533495, esp = 0xcbaa582c, ebp = 0xcbaa5854 ---
witness_checkorder(c111776c,9,c06b597b,2bb) at witness_checkorder+0x191
_mtx_lock_flags(c111776c,0,c06b5972,2bb,c10e3a80) at _mtx_lock_flags+0x5b
trap_pfault(cbaa58e4,0,608) at trap_pfault+0xb4
trap(c10e0018,c0750010,c0750010,9,c111776c) at trap+0x335
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc0533495, esp = 0xcbaa5924, ebp = 0xcbaa594c ---
witness_checkorder(c111776c,9,c06b597b,2bb) at witness_checkorder+0x191
_mtx_lock_flags(c111776c,0,c06b5972,2bb,c10e3a80) at _mtx_lock_flags+0x5b
trap_pfault(cbaa59dc,0,608) at trap_pfault+0xb4
trap(18,c10e0010,cbaa0010,9,c06fa2e0) at trap+0x335
calltrap() at calltrap+0x5
--- trap 0xc, eip = 0xc0533495, esp = 0xcbaa5a1c, ebp = 0xcbaa5a44 ---
witness_checkorder(c06fa2e0,9,c0697593,1d3) at witness_checkorder+0x191
_mtx_lock_flags(c06fa2e0,0,c069758a,1d3,c069851a) at _mtx_lock_flags+0x5b
alq_post(c13f3300,c15f5638) at alq_post+0x60
ktr_tracepoint(200,c0698511,186,c06985f6,c0674241,19,c10e2a08,0,0,0) at 
ktr_tracepoint+0x17e
ithread_schedule(c10d7900,e,c10e3a80,c15e7420,c06f00d0) at ithread_schedule+0x6b
intr_execute_handlers(c06f00d0,cbaa5b50,c15e7420,5fb,c10e3af0) at 
intr_execute_handlers+0x131
atpic_handle_intr(e) at atpic_handle_intr+0x92
Xatpic_intr14() at Xatpic_intr14+0x20
--- interrupt, eip = 0xc053392a, esp = 0xcbaa5b94, ebp = 0xcbaa5ba4 ---
witness_lock(c15e7420,8,c06a219e,5fb,c6342b10) at witness_lock+0xc2
_mtx_lock_flags(c15e7420,0,c06a2195,5fb) at _mtx_lock_flags+0x97
bqrelse(c6342b10) at bqrelse+0xe7
bufdone(c6342b10) at bufdone+0x446
cluster_callback(c62e64c0) at cluster_callback+0xa6
bufdone(c62e64c0,c0709820,0,c06a2195,c14) at bufdone+0x13b
bufdonebio(c62e64c0) at bufdonebio+0x24
biodone(c62e64c0,c15da18c,c15da18c,cbaa5cd0,cbaa5cb0) at biodone+0x57
g_dev_done(c15da18c) at g_dev_done+0x5b
biodone(c15da18c,cbaa5cd0,0,c0695cbe,1c1) at biodone+0x57
g_io_schedule_up(c10e3a80) at g_io_schedule_up+0xb5
g_up_procbody(0,cbaa5d48,0,c04e4dbc,0) at g_up_procbody+0x32
fork_exit(c04e4dbc,0,cbaa5d48) at fork_exit+0xd4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xcbaa5d7c, ebp = 0 ---
db>  show pcpu
cpuid        = 0
curthread    = 0xc10e3a80: pid 3 "g_up"
curpcb       = 0xcbaa5da0
fpcurthread  = none
idlethread   = 0xc10da480: pid 11 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x28
spin locks held:
db> db> show witness
Sleep locks:
0 ATAPI CD bioqueue lock -- last acquired _at_ dev/ata/atapi-cd.c:1100
0 g_xdown -- last acquired _at_ geom/geom_io.c:392
4  ATA disk bioqueue lock -- last acquired _at_ dev/ata/ata-disk.c:236
4  ATA queue lock -- last acquired _at_ dev/ata/ata-queue.c:176
5   ATA state lock -- last acquired _at_ dev/ata/ata-queue.c:182
4  bio queue -- last acquired _at_ geom/geom_io.c:65
12 system map -- last acquired _at_ vm/vm_kern.c:296
13  kmem object -- last acquired _at_ vm/vm_kern.c:398
14   vm page queue mutex -- last acquired _at_ kern/vfs_bio.c:3217
15    CMAPCADDR12 -- last acquired _at_ i386/i386/pmap.c:2473
15    vnode interlock -- last acquired _at_ kern/vfs_bio.c:1531
16     cdev -- last acquired _at_ kern/kern_conf.c:81
16     vnode_free_list -- last acquired _at_ kern/vfs_subr.c:575
16     Syncer mtx -- last acquired _at_ kern/vfs_subr.c:1846
16     Name Cache -- last acquired _at_ kern/vfs_cache.c:352
15    pmap -- last acquired _at_ i386/i386/pmap.c:1901
16     uma object -- last acquired _at_ vm/uma_core.c:963
16     UMA pcpu -- last acquired _at_ vm/uma_core.c:2206
17      KMAP ENTRY -- last acquired _at_ vm/uma_core.c:2224
18       UMA zone -- last acquired _at_ vm/uma_core.c:2224
13  kernel object -- last acquired _at_ kern/vfs_bio.c:3671
14   vm page queue mutex -- (already displayed)
0 g_xup -- last acquired _at_ geom/geom_io.c:449
1  g_disk_done -- last acquired _at_ geom/geom_disk.c:196
4   bio queue -- (already displayed)
16  UMA pcpu -- (already displayed)
3  Giant -- last acquired _at_ kern/vfs_bio.c:3092
4   struct mount mtx -- last acquired _at_ ufs/ffs/ffs_vfsops.c:1147
15   vnode interlock -- (already displayed)
4   UMA lock -- last acquired _at_ vm/uma_core.c:1466
12   Malloc Stats -- last acquired _at_ kern/kern_malloc.c:210
16   UMA pcpu -- (already displayed)
4   eventhandler -- last acquired _at_ kern/subr_eventhandler.c:213
5    eventhandler list -- last acquired _at_ kern/kern_exit.c:199
4   standard object -- last acquired _at_ kern/vfs_bio.c:3198
5    vm object_list -- last acquired _at_ vm/vm_object.c:643
14   vm page queue mutex -- (already displayed)
4   kernel linker -- last acquired _at_ kern/kern_linker.c:1061
4   vm86 lock -- last acquired _at_ i386/i386/vm86.c:582
4   TID lock -- last acquired _at_ kern/kern_thread.c:206
4   kobj -- last acquired _at_ kern/subr_kobj.c:298
4   GEOM orphanage -- last acquired _at_ geom/geom_event.c:170
4   ithread -- last acquired _at_ kern/kern_intr.c:276
4   bounce pages lock -- last acquired _at_ i386/i386/busdma_machdep.c:860
4   bpf global lock -- last acquired _at_ net/bpf.c:1446
5    bpf interface lock -- last acquired _at_ order list:0
6     bpf cdev lock -- last acquired _at_ order list:0
4   rman -- last acquired _at_ kern/subr_rman.c:448
12   Malloc Stats -- (already displayed)
12   system map -- (already displayed)
4   taskqueue list -- last acquired _at_ kern/subr_taskqueue.c:85
4   rman head -- last acquired _at_ kern/subr_rman.c:111
4   sf_buf -- last acquired _at_ i386/i386/vm_machdep.c:674
4   bio queue -- (already displayed)
4   rtsock route_cb lock -- last acquired _at_ net/rtsock.c:234
4   devstat -- last acquired _at_ kern/subr_devstat.c:190
4   if_cloners lock -- last acquired _at_ net/if_clone.c:199
4   buffer daemon lock -- last acquired _at_ kern/vfs_bio.c:408
4   pseudofs -- last acquired _at_ fs/pseudofs/pseudofs_fileno.c:86
4   ttylist -- last acquired _at_ kern/tty.c:2757
11   tty -- last acquired _at_ kern/kern_event.c:1451
4   fdesc -- last acquired _at_ kern/kern_descrip.c:1614
5    filedesc structure -- last acquired _at_ kern/kern_descrip.c:1926
6     devd -- last acquired _at_ kern/subr_bus.c:497
9      sellck -- last acquired _at_ kern/sys_generic.c:726
6     accept -- last acquired _at_ kern/uipc_socket.c:334
7      so_snd -- last acquired _at_ kern/uipc_socket.c:2091
8       so_rcv -- last acquired _at_ kern/uipc_socket.c:2092
9        sellck -- (already displayed)
9        radix node head -- last acquired _at_ netinet/if_ether.c:139
10        ifnet -- last acquired _at_ net/if.c:1019
10        rtentry -- last acquired _at_ netinet/if_ether.c:785
11         rts_inq -- last acquired _at_ net/netisr.c:231
11         network driver -- last acquired _at_ dev/fxp/if_fxp.c:1756
12          knlist lock for lockless objects -- last acquired _at_ 
kern/kern_event.c:1451
12          Malloc Stats -- (already displayed)
12          if send queue -- last acquired _at_ dev/fxp/if_fxp.c:1268
12          system map -- (already displayed)
11         ifaddr -- last acquired _at_ netinet6/nd6_nbr.c:1235
9        process lock -- last acquired _at_ ufs/ffs/ffs_vnops.c:613
10        sigacts -- last acquired _at_ kern/kern_sig.c:294
10        ktrace -- last acquired _at_ kern/kern_fork.c:607
10        struct pargs.ref -- last acquired _at_ kern/kern_proc.c:1078
10        session -- last acquired _at_ kern/kern_proc.c:317
11         tty -- (already displayed)
15         vnode interlock -- (already displayed)
11         uidinfo hash -- last acquired _at_ kern/kern_resource.c:1001
12          sleep mtxpool -- last acquired _at_ kern/kern_prot.c:1755
12          uidinfo struct -- last acquired _at_ order list:0
13           allprison -- last acquired _at_ kern/kern_jail.c:460
6     pipe mutex -- last acquired _at_ kern/sys_pipe.c:1520
9      sellck -- (already displayed)
7      sigio lock -- last acquired _at_ kern/kern_descrip.c:729
8       process group -- last acquired _at_ kern/kern_proc.c:393
9        process lock -- (already displayed)
4   if_clone lock -- last acquired _at_ net/if_clone.c:321
4   pfil_head_mtx -- last acquired _at_ net/pfil.c:166
5    pfil_head_list lock -- last acquired _at_ net/pfil.c:172
4   lo_mtx -- last acquired _at_ net/if_loop.c:154
4   nfsd_mtx -- last acquired _at_ nfsserver/nfs_srvsock.c:811
4   pbuf mutex -- last acquired _at_ vm/vm_pager.c:414
4   domain list -- last acquired _at_ kern/uipc_domain.c:110
4   mntid -- last acquired _at_ kern/vfs_subr.c:400
5    mountlist -- last acquired _at_ kern/vfs_subr.c:3456
4   ATA disk bioqueue lock -- (already displayed)
4   ATA queue lock -- (already displayed)
4   taskqueue -- last acquired _at_ kern/subr_taskqueue.c:193
4   pseudofs_vncache -- last acquired _at_ fs/pseudofs/pseudofs_vncache.c:239
4   udp -- last acquired _at_ netinet/udp_usrreq.c:1052
5    udp6inp -- last acquired _at_ netinet6/udp6_usrreq.c:670
8     so_rcv -- (already displayed)
5    udpinp -- last acquired _at_ netinet/udp_usrreq.c:1058
6     arc4_mtx -- last acquired _at_ libkern/arc4random.c:137
6     accept -- (already displayed)
4   ufs ihash -- last acquired _at_ ufs/ufs/ufs_ihash.c:120
15   vnode interlock -- (already displayed)
4   dirhash list -- last acquired _at_ ufs/ufs/ufs_dirhash.c:348
5    dirhash -- last acquired _at_ ufs/ufs/ufs_dirhash.c:349
4   runningbufspace lock -- last acquired _at_ kern/vfs_bio.c:317
5    ALDmtx -- last acquired _at_ kern/kern_alq.c:195
4   bdone lock -- last acquired _at_ kern/vfs_bio.c:3012
4   needsbuffer lock -- last acquired _at_ kern/vfs_bio.c:298
4   buf queue lock -- last acquired _at_ kern/vfs_bio.c:1520
15   vnode interlock -- (already displayed)
4   accounting -- last acquired _at_ kern/kern_acct.c:232
4   tcp -- last acquired _at_ netinet/tcp_timer.c:138
5    tcpinp -- last acquired _at_ netinet/tcp_timer.c:184
6     so_glabel -- last acquired _at_ kern/uipc_socket.c:282
6     tcp_hc_entry -- last acquired _at_ netinet/tcp_hostcache.c:713
6     random reseed -- last acquired _at_ dev/random/yarrow.c:193
6     arc4_mtx -- (already displayed)
6     accept -- (already displayed)
4   protect sysfilt_ops -- last acquired _at_ kern/kern_event.c:667
4   unp -- last acquired _at_ kern/uipc_usrreq.c:247
6    accept -- (already displayed)
0 arp_inq -- last acquired _at_ net/netisr.c:231
0 ip6qlock -- last acquired _at_ netinet6/frag6.c:682
0 igmp_mtx -- last acquired _at_ netinet/igmp.c:431
0 ip_inq -- last acquired _at_ net/netisr.c:231
0 ipqlock -- last acquired _at_ netinet/ip_input.c:1096
0 sem -- last acquired _at_ kern/sysv_sem.c:1174
0 if_afdata -- last acquired _at_ net/if.c:489
0 GEOM topology -- last acquired _at_ geom/geom_event.c:202
4  bio queue -- (already displayed)
4  GEOM orphanage -- (already displayed)
4  bdone lock -- (already displayed)
12 Malloc Stats -- (already displayed)
4  devstat -- (already displayed)
12 system map -- (already displayed)
4  ATA queue lock -- (already displayed)
1  swapdev -- last acquired _at_ vm/swap_pager.c:2127
0 p_peers -- last acquired _at_ kern/kern_exit.c:243
0 module subsystem sx lock -- last acquired _at_ kern/kern_module.c:404
0 rawcb -- last acquired _at_ net/raw_usrreq.c:80
8  so_rcv -- (already displayed)
0 kernel environment -- last acquired _at_ kern/kern_environment.c:285
0 sysctl lock -- last acquired _at_ kern/kern_sysctl.c:1315
1  addrsel_lock -- last acquired _at_ netinet6/in6_src.c:1137
1  malloc -- last acquired _at_ kern/kern_malloc.c:588
1  rip -- last acquired _at_ netinet/raw_ip.c:845
1  filelist lock -- last acquired _at_ kern/kern_descrip.c:1388
5   filedesc structure -- (already displayed)
1  allproc -- last acquired _at_ kern/sched_4bsd.c:428
2   user map -- last acquired _at_ vm/vm_glue.c:175
3    Giant -- (already displayed)
0 ddp_list_mtx -- last acquired _at_ order list:0
1  ddp_mtx -- last acquired _at_ order list:0
0 slip_mtx -- last acquired _at_ order list:0
1  slip sc_mtx -- last acquired _at_ order list:0
0 proctree -- last acquired _at_ kern/kern_prot.c:401
1  allproc -- (already displayed)

Spin locks:

Locks which were never acquired:
swap_pager swhash
scope6_lock
ip6_inq
pseudofs_fileno
tunmtx
msq
semid
msdosfs dehash
nfs4dev state
nfs4dev waitq
nfs4dev newq
cd9660_ihash
agp lock
callout_wait_lock
fifo mutex
UUID generator mutex lock
umtx
jumbo mutex
accept_filter_mtx
encapmtx
securelevel mutex lock
kqueue order
phys_pager list
dev_pager list
dev_pager create
swap_pager list
vm map sleep mutex
lockmgr
PMAP2
db>
Received on Mon Sep 27 2004 - 04:15:09 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:38:14 UTC