ipfw: LOR/panic with uid rules

Hello,

Also posted about this problem recently in stable_at_. But got no replies there. 
So I tried on a recent CURRENT but the problem persists:

ipfw rules using uid are causing a deadlock.
eg. allow ip from any to any uid root
A simple HTTP fetch triggers this problem nearly instantly.

For me, this problem existed in 6.x with PREEMPTION enabled. It was fixed in 
7.0. But in RELENG_7 and head it's back. This is a single processor i386 
machine.

With INVARIANTS/WITNESS there is some hopefully useful debug output.

lock order reversal:
 1st 0xc103d96c IPFW static rules (IPFW static rules) _at_ 
/usr/src/sys/modules/ipfw/../../netinet/ip_fw2.c:2473
 2nd 0xc0e5aaec udp (udp) _at_ 
/usr/src/sys/modules/ipfw/../../netinet/ip_fw2.c:2020
KDB: stack backtrace:
db_trace_self_wrapper(c0bad113,c47326d0,c082ccf5,4,c0ba8abc,...) at 
db_trace_self_wrapper+0x26
kdb_backtrace(4,c0ba8abc,c103beed,c48749c0,c473272c,...) at kdb_backtrace+0x29
_witness_debugger(c0baf9c8,c0e5aaec,c0bc8710,c48749c0,c103beed,...) at 
_witness_debugger+0x25
witness_checkorder(c0e5aaec,1,c103beed,7e4,0,...) at witness_checkorder+0x810
_rw_rlock(c0e5aaec,c103beed,7e4,c47327a4,c082de43,...) at _rw_rlock+0x9c
ipfw_chk(c4732a7c,41ec0d7e,0,0,c4dc6000,...) at ipfw_chk+0x36ea
ipfw_check_in(0,c4732ba0,c4b0a000,1,0,...) at ipfw_check_in+0xe1
pfil_run_hooks(c0e599c0,c4732bf4,c4b0a000,1,0,...) at pfil_run_hooks+0x98
ip_input(c4dc6000,b395eb11,800,c4b0a000,800,...) at ip_input+0x24d
netisr_dispatch(2,c4dc6000,10,3,0,...) at netisr_dispatch+0x73
ether_demux(c4b0a000,c4dc6000,3,0,3,...) at ether_demux+0x1f1
ether_input(c4b0a000,c4dc6000,c0b9dd13,585,c0cf63c0,...) at ether_input+0x37f
vr_intr(c4b22000,c4732cc8,c07e0c54,c0cf63c0,c4905ab8,...) at vr_intr+0x49e
intr_event_execute_handlers(c48c07d4,c4905a80,c0ba669c,4dd,c4905af0,...) at 
intr_event_execute_handlers+0x125
ithread_loop(c4b29a10,c4732d38,c0ba640e,322,c48c07d4,...) at ithread_loop+0x9f
fork_exit(c07d06b0,c4b29a10,c4732d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc4732d70, ebp = 0 ---
lock order reversal:
 1st 0xc103d96c IPFW static rules (IPFW static rules) _at_ 
/usr/src/sys/modules/ipfw/../../netinet/ip_fw2.c:2473
 2nd 0xc0e5a6ec tcp (tcp) _at_ 
/usr/src/sys/modules/ipfw/../../netinet/ip_fw2.c:2020
KDB: stack backtrace:
db_trace_self_wrapper(c0bad113,c47326d0,c082ccf5,4,c0ba8abc,...) at 
db_trace_self_wrapper+0x26
kdb_backtrace(4,c0ba8abc,c103beed,c4874a28,c473272c,...) at kdb_backtrace+0x29
_witness_debugger(c0baf9c8,c0e5a6ec,c0bafeac,c4874a28,c103beed,...) at 
_witness_debugger+0x25
witness_checkorder(c0e5a6ec,1,c103beed,7e4,0,...) at witness_checkorder+0x810
_rw_rlock(c0e5a6ec,c103beed,7e4,a00a8c0,97e7,...) at _rw_rlock+0x9c
ipfw_chk(c4732a7c,41ec0d7e,0,0,c4dc6200,...) at ipfw_chk+0x36ea
ipfw_check_in(0,c4732ba0,c4b0a000,1,0,...) at ipfw_check_in+0xe1
pfil_run_hooks(c0e599c0,c4732bf4,c4b0a000,1,0,...) at pfil_run_hooks+0x98
ip_input(c4dc6200,b395eb11,800,c4b0a000,800,...) at ip_input+0x24d
netisr_dispatch(2,c4dc6200,10,3,0,...) at netisr_dispatch+0x73
ether_demux(c4b0a000,c4dc6200,3,0,3,...) at ether_demux+0x1f1
ether_input(c4b0a000,c4dc6200,c0b9dd13,585,c0cf63c0,...) at ether_input+0x37f
vr_intr(c4b22000,c4732cc8,c07e0c54,c0cf63c0,c4905ab8,...) at vr_intr+0x49e
intr_event_execute_handlers(c48c07d4,c4905a80,c0ba669c,4dd,c4905af0,...) at 
intr_event_execute_handlers+0x125
ithread_loop(c4b29a10,c4732d38,c0ba640e,322,c48c07d4,...) at ithread_loop+0x9f
fork_exit(c07d06b0,c4b29a10,c4732d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xc4732d70, ebp = 0 ---

If I hit CTRL+C to cancel the fetch, I get this panic:

Unread portion of the kernel message buffer:
panic: _rw_rlock (tcp): wlock already held _at_ 
/usr/src/sys/modules/ipfw/../../netinet/ip_fw2.c:2020

(kgdb) bt
#0  doadump () at pcpu.h:221
#1  0xc07ee2de in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:418
#2  0xc07ee5a3 in panic (fmt=Variable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:572
#3  0xc07eca66 in _rw_rlock (rw=0xc0e5a6ec, 
    file=0xc103beed "/usr/src/sys/modules/ipfw/../../netinet/ip_fw2.c", 
    line=2020) at /usr/src/sys/kern/kern_rwlock.c:283
#4  0xc103a92a in ipfw_chk (args=0xc4732828)
    at /usr/src/sys/modules/ipfw/../../netinet/ip_fw2.c:2020
#5  0xc103b4c8 in ipfw_check_out (arg=0x0, m0=0xc473294c, ifp=0xc4b0a000, 
    dir=2, inp=0x0)
    at /usr/src/sys/modules/ipfw/../../netinet/ip_fw_pfil.c:253
#6  0xc0898f28 in pfil_run_hooks (ph=0xc0e599c0, mp=0xc47329bc, 
    ifp=0xc4b0a000, dir=2, inp=0x0) at /usr/src/sys/net/pfil.c:79
#7  0xc08e0f32 in ip_output (m=0xc4dce000, opt=0x0, ro=0xc47329c4, flags=0, 
    imo=0x0, inp=0x0) at /usr/src/sys/netinet/ip_output.c:452
#8  0xc0943cd5 in tcp_respond (tp=0x0, ipgen=0xc4e0e016, th=0xc4e0e02a, 
    m=0xc4dce000, ack=0, seq=1292138936, flags=Variable "flags" is not 
available.
)
    at /usr/src/sys/netinet/tcp_subr.c:611
#9  0xc093a8c5 in tcp_dropwithreset (m=0xc4dce000, th=0xc4e0e02a, tp=0x0, 
    tlen=1440, rstreason=-1) at /usr/src/sys/netinet/tcp_input.c:2545
#10 0xc093c863 in tcp_do_segment (m=0xc4dce000, th=0xc4e0e02a, so=0xc4fd4000, 
    tp=0x0, drop_hdrlen=52, tlen=1440, iptos=0 '\0')
    at /usr/src/sys/netinet/tcp_input.c:2475
#11 0xc093d71c in tcp_input (m=0xc4dce000, off0=20)
    at /usr/src/sys/netinet/tcp_input.c:882
#12 0xc08df540 in ip_input (m=0xc4dce000)
    at /usr/src/sys/netinet/ip_input.c:666
#13 0xc0898723 in netisr_dispatch (num=2, m=0xc4dce000)
    at /usr/src/sys/net/netisr.c:178
#14 0xc0892671 in ether_demux (ifp=0xc4b0a000, m=0xc4dce000)
    at /usr/src/sys/net/if_ethersubr.c:842
#15 0xc0892adf in ether_input (ifp=0xc4b0a000, m=0xc4dce000)
    at /usr/src/sys/net/if_ethersubr.c:700
#16 0xc0764e3e in vr_intr (arg=0xc4b22000) at /usr/src/sys/dev/vr/if_vr.c:1414
#17 0xc07cfad5 in intr_event_execute_handlers (p=0xc48c07d4, ie=0xc4905a80)
    at /usr/src/sys/kern/kern_intr.c:1134
#18 0xc07d074f in ithread_loop (arg=0xc4b29a10)
    at /usr/src/sys/kern/kern_intr.c:1147
#19 0xc07cd898 in fork_exit (callout=0xc07d06b0 <ithread_loop>, 
    arg=0xc4b29a10, frame=0xc4732d38) at /usr/src/sys/kern/kern_fork.c:810
#20 0xc0ae34d0 in fork_trampoline () at /usr/src/sys/i386/i386/exception.s:270