PF NAT issue with 9.0-BETA3 and RELENG_9 'head'

Hello,

i recently switched a router in our test-environment to FreeBSD 9.0-Beta3
(and after things didnt worked ... checked out the current RELENG_9
and recompiled kernel & world .. )



Problem:
 After 5 - 15 minutes NAT stops working (normal routing still works.)

 Network Utilization:  about 40 MByte/second, which gets routed
  only a few kbit/s are getting natted (NTP Syncs and such ... )

  When i took a look on the nat rules (via pfctl -vv -s nat)
  the rules gets evaluated; but nothing matches anymore...

  State Table helds about 9500 Entrys,
  Source Tracking Table about 300





Software / Configuration:
 pf, carp

 pf.conf:
====================================================
set limit src-nodes 550000
set limit frags 32000
set timeout { adaptive.start 530000 adaptive.end 540000 }
set timeout src.track 600
set timeout frag 30

set skip on lo0
set skip on igb2
set skip on igb3
set skip on bce0
set skip on bce1
set skip on pfsync0
#set skip on internal
#set skip on carp3internal

nat on public from 10.5.0.0/16 to any -> { public }
====================================================

carp device holding the internal gateway ips (10.5.0.253 .. ),
currently master - no slave


/etc/sysctl.conf:
====================================================
net.inet.tcp.blackhole=2
net.inet.udp.blackhole=1
net.inet.ip.forwarding=1
net.inet.ip.fastforwarding=1
net.inet.icmp.icmplim_output=0
net.inet.icmp.icmplim=0
net.route.netisr_maxqlen=8192
kern.random.sys.harvest.interrupt=0
kern.random.sys.harvest.ethernet=0
kern.random.sys.harvest.point_to_point=0
net.inet.carp.preempt=1
====================================================


/boot/loader.conf:
====================================================
net.isr.maxthreads="2"
net.isr.defaultqlimit="4096"
net.isr.maxqlimit="81920"
net.isr.direct="1"
net.isr.bindthreads="1"

hw.igb.num_queues=2
hw.igb.enable_aim=1
hw.igb.txd=2048
hw.igb.rxd=2048
hw.igb.max_interrupt_rate=8000
hw.intr_storm_threshold=10000

kern.ipc.nmbclusters="262144"

kern.hz=1000
====================================================


# sysctl -a hw.igb
hw.igb.rx_process_limit: 100
hw.igb.num_queues: 2
hw.igb.header_split: 0
hw.igb.max_interrupt_rate: 8000
hw.igb.enable_msix: 1
hw.igb.enable_aim: 1
hw.igb.txd: 2048
hw.igb.rxd: 2048


# sysctl -a dev.igb
dev.igb.0.%desc: Intel(R) PRO/1000 Network Connection version - 2.2.5
dev.igb.0.%driver: igb
dev.igb.0.%location: slot=0 function=0
dev.igb.0.%pnpinfo: vendor=0x8086 device=0x10e8 subvendor=0x8086
subdevice=0xa02c class=0x020000
dev.igb.0.%parent: pci5
dev.igb.0.nvm: -1
dev.igb.0.enable_aim: 1
dev.igb.0.fc: 65536003
dev.igb.0.rx_processing_limit: 100
dev.igb.0.link_irq: 2
dev.igb.0.dropped: 0
dev.igb.0.tx_dma_fail: 0
dev.igb.0.rx_overruns: 0
dev.igb.0.watchdog_timeouts: 0
dev.igb.0.device_control: 1086325313
dev.igb.0.rx_control: 67141634
dev.igb.0.interrupt_mask: 4
dev.igb.0.extended_int_mask: 2147483655
dev.igb.0.tx_buf_alloc: 0
dev.igb.0.rx_buf_alloc: 0
dev.igb.0.fc_high_water: 58976
dev.igb.0.fc_low_water: 58960
dev.igb.0.queue0.no_desc_avail: 0
dev.igb.0.queue0.tx_packets: 28167655
dev.igb.0.queue0.rx_packets: 942710
dev.igb.0.queue0.rx_bytes: 84905673
dev.igb.0.queue0.lro_queued: 0
dev.igb.0.queue0.lro_flushed: 0
dev.igb.0.queue1.no_desc_avail: 0
dev.igb.0.queue1.tx_packets: 27659961
dev.igb.0.queue1.rx_packets: 219218
dev.igb.0.queue1.rx_bytes: 34229378
dev.igb.0.queue1.lro_queued: 0
dev.igb.0.queue1.lro_flushed: 0
dev.igb.0.mac_stats.excess_coll: 0
dev.igb.0.mac_stats.single_coll: 0
dev.igb.0.mac_stats.multiple_coll: 0
dev.igb.0.mac_stats.late_coll: 0
dev.igb.0.mac_stats.collision_count: 0
dev.igb.0.mac_stats.symbol_errors: 0
dev.igb.0.mac_stats.sequence_errors: 0
dev.igb.0.mac_stats.defer_count: 0
dev.igb.0.mac_stats.missed_packets: 0
dev.igb.0.mac_stats.recv_no_buff: 0
dev.igb.0.mac_stats.recv_undersize: 0
dev.igb.0.mac_stats.recv_fragmented: 0
dev.igb.0.mac_stats.recv_oversize: 0
dev.igb.0.mac_stats.recv_jabber: 0
dev.igb.0.mac_stats.recv_errs: 0
dev.igb.0.mac_stats.crc_errs: 0
dev.igb.0.mac_stats.alignment_errs: 0
dev.igb.0.mac_stats.coll_ext_errs: 0
dev.igb.0.mac_stats.xon_recvd: 0
dev.igb.0.mac_stats.xon_txd: 0
dev.igb.0.mac_stats.xoff_recvd: 0
dev.igb.0.mac_stats.xoff_txd: 0
dev.igb.0.mac_stats.total_pkts_recvd: 1277070
dev.igb.0.mac_stats.good_pkts_recvd: 1161923
dev.igb.0.mac_stats.bcast_pkts_recvd: 101354
dev.igb.0.mac_stats.mcast_pkts_recvd: 714
dev.igb.0.mac_stats.rx_frames_64: 102154
dev.igb.0.mac_stats.rx_frames_65_127: 1015473
dev.igb.0.mac_stats.rx_frames_128_255: 6736
dev.igb.0.mac_stats.rx_frames_256_511: 10919
dev.igb.0.mac_stats.rx_frames_512_1023: 1719
dev.igb.0.mac_stats.rx_frames_1024_1522: 24922
dev.igb.0.mac_stats.good_octets_recvd: 123782443
dev.igb.0.mac_stats.good_octets_txd: 55500343847
dev.igb.0.mac_stats.total_pkts_txd: 55828073
dev.igb.0.mac_stats.good_pkts_txd: 55828073
dev.igb.0.mac_stats.bcast_pkts_txd: 5
dev.igb.0.mac_stats.mcast_pkts_txd: 1
dev.igb.0.mac_stats.tx_frames_64: 10267735
dev.igb.0.mac_stats.tx_frames_65_127: 4630167
dev.igb.0.mac_stats.tx_frames_128_255: 756857
dev.igb.0.mac_stats.tx_frames_256_511: 3548802
dev.igb.0.mac_stats.tx_frames_512_1023: 1936496
dev.igb.0.mac_stats.tx_frames_1024_1522: 34688016
dev.igb.0.mac_stats.tso_txd: 452
dev.igb.0.mac_stats.tso_ctx_fail: 0
dev.igb.0.interrupts.asserts: 22013222
dev.igb.0.interrupts.rx_pkt_timer: 1161904
dev.igb.0.interrupts.rx_abs_timer: 0
dev.igb.0.interrupts.tx_pkt_timer: 0
dev.igb.0.interrupts.tx_abs_timer: 1161923
dev.igb.0.interrupts.tx_queue_empty: 55827161
dev.igb.0.interrupts.tx_queue_min_thresh: 0
dev.igb.0.interrupts.rx_desc_min_thresh: 0
dev.igb.0.interrupts.rx_overrun: 0
dev.igb.0.host.breaker_tx_pkt: 0
dev.igb.0.host.host_tx_pkt_discard: 0
dev.igb.0.host.rx_pkt: 19
dev.igb.0.host.breaker_rx_pkts: 0
dev.igb.0.host.breaker_rx_pkt_drop: 0
dev.igb.0.host.tx_good_pkt: 912
dev.igb.0.host.breaker_tx_pkt_drop: 0
dev.igb.0.host.rx_good_bytes: 123782443
dev.igb.0.host.tx_good_bytes: 55500343847
dev.igb.0.host.length_errors: 0
dev.igb.0.host.serdes_violation_pkt: 0
dev.igb.0.host.header_redir_missed: 0
dev.igb.1.%desc: Intel(R) PRO/1000 Network Connection version - 2.2.5
dev.igb.1.%driver: igb
dev.igb.1.%location: slot=0 function=1
dev.igb.1.%pnpinfo: vendor=0x8086 device=0x10e8 subvendor=0x8086
subdevice=0xa02c class=0x020000
dev.igb.1.%parent: pci5
dev.igb.1.nvm: -1
dev.igb.1.enable_aim: 1
dev.igb.1.fc: 65536003
dev.igb.1.rx_processing_limit: 100
dev.igb.1.link_irq: 2
dev.igb.1.dropped: 0
dev.igb.1.tx_dma_fail: 0
dev.igb.1.rx_overruns: 0
dev.igb.1.watchdog_timeouts: 0
dev.igb.1.device_control: 1086325313
dev.igb.1.rx_control: 67141658
dev.igb.1.interrupt_mask: 4
dev.igb.1.extended_int_mask: 2147483655
dev.igb.1.tx_buf_alloc: 0
dev.igb.1.rx_buf_alloc: 0
dev.igb.1.fc_high_water: 58976
dev.igb.1.fc_low_water: 58960
dev.igb.1.queue0.no_desc_avail: 0
dev.igb.1.queue0.tx_packets: 863716
dev.igb.1.queue0.rx_packets: 28455079
dev.igb.1.queue0.rx_bytes: 28046622063
dev.igb.1.queue0.lro_queued: 0
dev.igb.1.queue0.lro_flushed: 0
dev.igb.1.queue1.no_desc_avail: 0
dev.igb.1.queue1.tx_packets: 232166
dev.igb.1.queue1.rx_packets: 27840375
dev.igb.1.queue1.rx_bytes: 27298049141
dev.igb.1.queue1.lro_queued: 0
dev.igb.1.queue1.lro_flushed: 0
dev.igb.1.mac_stats.excess_coll: 0
dev.igb.1.mac_stats.single_coll: 0
dev.igb.1.mac_stats.multiple_coll: 0
dev.igb.1.mac_stats.late_coll: 0
dev.igb.1.mac_stats.collision_count: 0
dev.igb.1.mac_stats.symbol_errors: 0
dev.igb.1.mac_stats.sequence_errors: 0
dev.igb.1.mac_stats.defer_count: 0
dev.igb.1.mac_stats.missed_packets: 0
dev.igb.1.mac_stats.recv_no_buff: 0
dev.igb.1.mac_stats.recv_undersize: 0
dev.igb.1.mac_stats.recv_fragmented: 0
dev.igb.1.mac_stats.recv_oversize: 0
dev.igb.1.mac_stats.recv_jabber: 0
dev.igb.1.mac_stats.recv_errs: 0
dev.igb.1.mac_stats.crc_errs: 0
dev.igb.1.mac_stats.alignment_errs: 0
dev.igb.1.mac_stats.coll_ext_errs: 0
dev.igb.1.mac_stats.xon_recvd: 0
dev.igb.1.mac_stats.xon_txd: 0
dev.igb.1.mac_stats.xoff_recvd: 0
dev.igb.1.mac_stats.xoff_txd: 0
dev.igb.1.mac_stats.total_pkts_recvd: 56298320
dev.igb.1.mac_stats.good_pkts_recvd: 56295417
dev.igb.1.mac_stats.bcast_pkts_recvd: 100932
dev.igb.1.mac_stats.mcast_pkts_recvd: 109429
dev.igb.1.mac_stats.rx_frames_64: 10539600
dev.igb.1.mac_stats.rx_frames_65_127: 4789005
dev.igb.1.mac_stats.rx_frames_128_255: 758560
dev.igb.1.mac_stats.rx_frames_256_511: 3556870
dev.igb.1.mac_stats.rx_frames_512_1023: 1939210
dev.igb.1.mac_stats.rx_frames_1024_1522: 34712172
dev.igb.1.mac_stats.good_octets_recvd: 55569850268
dev.igb.1.mac_stats.good_octets_txd: 121738026
dev.igb.1.mac_stats.total_pkts_txd: 1095880
dev.igb.1.mac_stats.good_pkts_txd: 1095880
dev.igb.1.mac_stats.bcast_pkts_txd: 428
dev.igb.1.mac_stats.mcast_pkts_txd: 3494
dev.igb.1.mac_stats.tx_frames_64: 1961
dev.igb.1.mac_stats.tx_frames_65_127: 1037835
dev.igb.1.mac_stats.tx_frames_128_255: 17407
dev.igb.1.mac_stats.tx_frames_256_511: 12213
dev.igb.1.mac_stats.tx_frames_512_1023: 1853
dev.igb.1.mac_stats.tx_frames_1024_1522: 24611
dev.igb.1.mac_stats.tso_txd: 81
dev.igb.1.mac_stats.tso_ctx_fail: 0
dev.igb.1.interrupts.asserts: 22296050
dev.igb.1.interrupts.rx_pkt_timer: 56294394
dev.igb.1.interrupts.rx_abs_timer: 0
dev.igb.1.interrupts.tx_pkt_timer: 0
dev.igb.1.interrupts.tx_abs_timer: 56295417
dev.igb.1.interrupts.tx_queue_empty: 1095875
dev.igb.1.interrupts.tx_queue_min_thresh: 0
dev.igb.1.interrupts.rx_desc_min_thresh: 0
dev.igb.1.interrupts.rx_overrun: 0
dev.igb.1.host.breaker_tx_pkt: 0
dev.igb.1.host.host_tx_pkt_discard: 0
dev.igb.1.host.rx_pkt: 1023
dev.igb.1.host.breaker_rx_pkts: 0
dev.igb.1.host.breaker_rx_pkt_drop: 0
dev.igb.1.host.tx_good_pkt: 5
dev.igb.1.host.breaker_tx_pkt_drop: 0
dev.igb.1.host.rx_good_bytes: 55569850268
dev.igb.1.host.tx_good_bytes: 121738026
dev.igb.1.host.length_errors: 0
dev.igb.1.host.serdes_violation_pkt: 0
dev.igb.1.host.header_redir_missed: 0

(Port 2 && 3 stripped .. due to no connectivity/unused)





The Hardware:
 Dell R410,  Xeon E5640
 6GByte Memory (DDR3..)
 Intel Quad Port GBit Adapter (82576) [igb..]

 Port.0 Used => public / provider
 Port.1 Used => internal network (servers)


 About 500 Machines behind this router.






Has anything changed from 8.2 to 9.0 that i missed to consider in configuration?



Thanks,
Florian