LOR in schedcpu(), sched_4bsd.c

With today's CURRENT:

...
GEOM: create disk ad0 dp=0xc2a3a380
ad0: 38166MB <ST340016A> [77545/16/63] at ata0-master UDMA100
acd0: DVDROM <_NEC DV-5700B> at ata1-master UDMA33
Mounting root from ufs:/dev/ad0s1a
Loading configuration files.
Entropy harvesting: interrupts ethernet point_to_point.
Reseed type 1
Reseed finish
lock order reversal
 1st 0xc06bfa20 callout_dont_sleep (callout_dont_sleep) _at_ kern/kern_timeout.c:223
 2nd 0xc06bed80 allproc /sched_4bsd.c:253
Stack backtrace:
backtrace(c06537e4,c06bed80,c065018e,c065018e,c0651c30) at backtrace+0x17
witness_lock(c06bed80,0,c0651c30,fd,c06c1e60) at witness_lock+0x697
_sx_slock(c06bed80,c0651c27,fd,8,c0651945) at _sx_slock+0xa9
schedcpu(0,0,c065193c,df,c12a8ab0) at schedcpu+0x3f
softclock(0,0,c064e447,230,c12a77d0) at softclock+0x1fb
ithread_loop(c129d200,cdb17d48,c064e2c1,314,c70000e8) at ithread_loop+0x182
fork_exit(c04aa4a0,c129d200,cdb17d48) at fork_exit+0xc1
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xcdb17d7c, ebp = 0 ---
Debugger("witness_lock")
Stopped at      Debugger+0x54:  xchgl   %ebx,in_Debugger.0
db> show locks
exclusive sleep mutex callout_dont_sleep r = 0 (0xc06bfa20) locked _at_ kern/kern_timeout.c:223
db> show witness
Sleep locks:
0 taskqueue kthread -- last acquired _at_ kern/subr_taskqueue.c:253
0 g_xdown -- last acquired _at_ geom/geom_io.c:351
1  ATA disk bioqueue lock -- last acquired _at_ dev/ata/ata-disk.c:240
3  bio queue -- last acquired _at_ geom/geom_io.c:64
5  Malloc Stats -- last acquired _at_ kern/kern_malloc.c:335
3  ATA queue lock -- last acquired _at_ dev/ata/ata-queue.c:174
8  UMA pcpu -- last acquired _at_ vm/uma_core.c:1726
9   KMAP ENTRY -- last acquired _at_ vm/uma_core.c:1744
9   UMA zone -- last acquired _at_ vm/uma_core.c:1744
0 g_xup -- last acquired _at_ geom/geom_io.c:370
2  Giant -- last acquired _at_ kern/kern_timeout.c:216
3   malloc -- last acquired _at_ cquired _at_ kern/subr_eventhandler.c:213
4    eventhandler list -- last acquired _at_ kern/kern_exit.c:210
3   vm object_list -- last acquired _at_ vm/vm_object.c:620
3   arc4_mtx -- last acquired _at_ libkern/arc4random.c:137
3   UMA lock -- last acquired _at_ vm/uma_core.c:802
3   filedesc structure -- last acquired _at_ kern/kern_descrip.c:1625
4    pipe mutex -- last acquired _at_ kern/sys_pipe.c:481
5     sigio lock -- last acquired _at_ kern/kern_descrip.c:587
6      process group -- last acquired _at_ kern/kern_fork.c:575
7       process lock -- last acquired _at_ kern/kern_prot.c:1821
8        struct pargs.ref -- last acquired _at_ kern/kern_proc.c:1077
8        sigacts -- last acquired _at_ kern/kern_sig.c:596
8        vnode interlock -- last acquired _at_ kern/vfs_subr.c:2176
9         Syncer mtx -- last acquired _at_ kern/vfs_subr.c:1663
9         vnode_free_list -- last acquired _at_ kern/vfs_subr.c:923
9         spechash -- last acquired _at_ kern/vfs_subr.c:1989
8        ktrace -- last acquired _at_ kern/kern_fork.c:601
8        session -- last acquired _at_ kern/kern_fork.c:584
9         uidinfo hash -- last acquired _at_ kern/kern_resource.c:878
10         sleep mtxpool -- last acquired _at_ kern/kern_prot.c:1685
10         uidinfo struct -- last acquired _at_ order list:0
11          allprison -- last acquired _at_ order list:0
3   ithread -- last acquired _at_ kern/kern_intr.c:265
3   GEOM orphanage -- last acquired _at_ geom/geom_event.c:169
3   rman head -- last acquired _at_ kern/subr_rman.c:110
3   sf_bufs list lock -- last acquired _at_ i386/i386/vm_machdep.c:577
5    Malloc Stats -- (already displayed)
5    system map -- last acquired _at_ vm/vm_map.c:2904
6     kmem object -- last acquired _at_ vm/vm_kern.c:437
7      vm page queue mutex -- last acquired _at_ vm/vm_object.c:594
8       vnode interlock -- (already displayed)
8       UMA pcpu -- (already displayed)
7      CMAPCADDR12 -- last acquired _at_ i386/i386/pmap.c:2475
6     vm object -- last acquired _at_ vm/vm_object.c:433
7      vm page queue mutex -- (already displayed)
7      CMAPCADDR12 -- (already displayed)
3   taskqueue list -- last acquired _at_ kern/subr_taskqueue.c:384
3   kernel linker -- last acquire/kern_linker.c:460
3   rman -- last acquired _at_ kern/subr_rman.c:132
5    Malloc Stats -- (already displayed)
5    system map -- (already displayed)
3   sellck -- last acquired _at_ kern/sys_generic.c:1174
3   domain list -- last acquired _at_ kern/uipc_domain.c:114
3   devd -- last acquired _at_ kern/subr_bus.c:406
3   callout_dont_sleep -- last acquired _at_ kern/kern_timeout.c:223
4    ifnet -- last acquired _at_ net/if.c:1172
4    tcp -- last acquired _at_ netinet/tcp_timer.c:141
4    ipflow list head -- last acquired _at_ netinet/ip_flow.c:288
4    ipqlock -- last acquired _at_ netinet/ip_input.c:1237
4    mbuf PCPU list lock -- last acquired _at_ kern/subr_mbuf.c:926
5     mbuf subsystem general lists lock -- last acquired _at_ kern/subr_mbuf.c:676
3   vm86 lock -- last acquired _at_ i386/i386/vm86.c:606
3   ATA queue lock -- (already displayed)
3   mntvnode vfs_subr.c:1054
3   pseudofs -- last acquired _at_ fs/pseudofs/pseudofs_fileno.c:86
3   bpf global lock -- last acquired _at_ net/bpf.c:1383
3   IPFW static rules -- last acquired _at_ netinet/ip_fw2.c:2159
3   random reseed -- last acquired _at_ dev/random/yarrow.c:172
3   mntid -- last acquired _at_ kern/vfs_subr.c:586
4    mountlist -- last acquired _at_ kern/vfs_mount.c:774
3   bio queue -- (already displayed)
3   pbuf mutex -- last acquired _at_ vm/vm_pager.c:443
3   devstat -- last acquired _at_ kern/subr_devstat.c:83
3   radix node head -- last acquired _at_ net/route.c:544
4    rtentry -- last acquired _at_ netinet6/nd6_rtr.c:1502
5     Malloc Stats -- (already displayed)
5     ifaddr -- last acquired _at_ net/route.c:670
5     system map -- (already displayed)
4    ifnet -- (already displayed)
3   buffer daemon lock -- last acquired _at_ kern/vfs_bio.c:2103
3   needsbuffer lock -- last acquired _at_ kern/vfs_bio.c:291
3   ufs ihash -- last acquired _at_ ufs/ufs/ufs_ihash.c:160
8    vnode interlock -- (already displayed)
3   bdone lock -- last acquired _at_ kern/vfs_bio.c:3747
3   p_peers -- last acquired _at_ kern/kern_exit.c:252
3   runningbufspace lock -- last acquired _at_ kern/vfs_bio.c:309
3   buf queue lock -- last acquired _at_ kern/vfs_bio.c:1387
8    vnode interlock -- (already displayed)
3   pseudofs_vncache -- last acquired _at_ fs/pseudofs/pseudofs_vncache.c:225
3   sem -- last acquired _at_ kern/sysv_sem.c:1158
3   accounting -- last acquired _at_ kern/kern_acct.c:232
3   fdesc -- last acquired _at_ kern/kern_descrip.c:1515
0 g_disk_done -- last acquired _at_ geom/geom_disk.c:183
3  bio queue -- (already displayed)
8  UMA pcpu -- (already displayed)
0 taskqueue -- last acquired _at_ kern/subr_taskqueue.c:210
0 GEOM event stalling -- last acquired _at_ geom/geom_event.c:157
1  GEOM topology -- last acquired _at_ geom/geom_event.c:158
5   Malloc Stats -- (already displayed)
3   devstat -- (already displayed)
3   GEOM orphanage -- (already displayed)
3   bio queue -- (already displayed)
3   bdone lock -- (already displayed)
5   system map -- (already displayed)
2  Giant -- (already displayed)
0 module subsystem sx lock -- last acquired _at_ kern/kern_module.c:102
0 filelist lock -- last acquired _at_ kern/kern_descrip.c:1233
3  filedesc structure -- (already displayed)
0 kernel environment -- last acquired _at_ kern/kern_environment.c:288
0 sysctl lock -- last acquired _at_ kern/kern_sysctl.c:1273
1  allproc -- last acquired _at_ kern/kern_fork.c:299
2   Giant -- (already displayed)
0 proctree -- last acquired _at_ kern/kern_exit.c:573
1  allproc -- (already displayed)

Spin locks:

Locks which were never acquired:
ATAPI CD bioqueue lock
arp_inq
rip
div
udp
ip_inq
ip6_inq
pseudofs_fileno
IPFW dynamic rules
msq
semid
cd9660_ihash
dirhash list
msdosfs dehash
bpf interface lock
if send queue
network driver
netgraph netisr mutex
netgraph idhash mutex
netgraph nodelist mutex
netgraph types mutex
strategy
netisr locbounce pages lock
umtx
rtsock route_cb lock
securelevel mutex lock
jumbo mutex
UUID generator mutex lock
phys_pager list
dev_pager list
dev_pager create
swapdev
swap_pager list
vm map sleep mutex
lockmgr