Debugging with memguard...

Howdy,

I'm trying to track down a memory issue with IPsec in CURRENT.  Using
the TAHI test suite and ssh I can cause the kernel to panic with the
"Memory modified after free" message at will.  The problem is that the
memory that was touched is from a seemingly random part of the
system.  I have seen IPsec and non-IPsec related memory (ttys,
proc-sav, ipsecrequest, and xform etc.)  This makes using
DEBUG_MEMGUARD rather hard.  Can anyone give a better way of tracking
down what's going on?

Thanks,
George

PS Kernel Config attached but it's not very interesting I think.

#
# VMWARE IPSEC -- Kernel config for VMWare VMs 4.5-5.5
#
# For more information on this file, please read the handbook section on
# Kernel Configuration Files:
#
#    http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
#
# The handbook is also available locally in /usr/share/doc/handbook
# if you've installed the doc distribution, otherwise always see the
# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the
# latest information.
#
# An exhaustive list of options and more detailed explanations of the
# device lines is also present in the ../../conf/NOTES and NOTES files.
# If you are in doubt as to the purpose or necessity of a line, check first
# in NOTES.
#
# $FreeBSD: src/sys/i386/conf/GENERIC,v 1.438 2006/01/21 12:38:35 marius Exp $

cpu		I686_CPU
ident		VMWARE_IPSEC

# To statically compile in device wiring instead of /boot/device.hints
#hints		"GENERIC.hints"		# Default places to look for devices.

makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols

#options 	SCHED_ULE		# ULE scheduler
options 	SCHED_4BSD		# 4BSD scheduler
options		DEBUG_MEMGUARD		# Debug memory issues
options 	PREEMPTION		# Enable kernel thread preemption
options 	INET			# InterNETworking
options         IPSEC			# IPSec
options		INET6			# IPv6 Support
options 	FFS			# Berkeley Fast Filesystem
options 	SOFTUPDATES		# Enable FFS soft updates support
options 	UFS_ACL			# Support for access control lists
options 	UFS_DIRHASH		# Improve performance on big directories
options 	MD_ROOT			# MD is a potential root device
options 	NFSCLIENT		# Network Filesystem Client
options 	NFSSERVER		# Network Filesystem Server
options 	NFS_ROOT		# NFS usable as /, requires NFSCLIENT
options 	MSDOSFS			# MSDOS Filesystem
options 	CD9660			# ISO 9660 Filesystem
options 	PROCFS			# Process filesystem (requires PSEUDOFS)
options 	PSEUDOFS		# Pseudo-filesystem framework
options 	COMPAT_43		# Compatible with BSD 4.3 [KEEP THIS!]
options 	COMPAT_43TTY		# BSD 4.3 TTY compat [KEEP THIS!]
options 	COMPAT_FREEBSD4		# Compatible with FreeBSD4
options 	COMPAT_FREEBSD5		# Compatible with FreeBSD5
options 	SCSI_DELAY=5000		# Delay (in ms) before probing SCSI
options 	KTRACE			# ktrace(1) support
options 	SYSVSHM			# SYSV-style shared memory
options 	SYSVMSG			# SYSV-style message queues
options 	SYSVSEM			# SYSV-style semaphores
options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options 	KBD_INSTALL_CDEV	# install a CDEV entry in /dev
options 	AHC_REG_PRETTY_PRINT	# Print register bitfields in debug
					# output.  Adds ~128k to driver.
options 	AHD_REG_PRETTY_PRINT	# Print register bitfields in debug
					# output.  Adds ~215k to driver.
options 	ADAPTIVE_GIANT		# Giant mutex is adaptive.
options 	STOP_NMI		# Stop CPUS using NMI instead of IPI

# Debugging for use in -current
options 	KDB			# Enable kernel debugger support.
options 	DDB			# Support DDB.
options 	GDB			# Support remote GDB.
options 	INVARIANTS		# Enable calls of extra sanity checking
options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
options 	WITNESS			# Enable checks to detect deadlocks and cycles
options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed

# To make an SMP kernel, the next two lines are needed
options 	SMP			# Symmetric MultiProcessor Kernel
device		apic			# I/O APIC

# Bus support.
device		eisa
device		pci

# ATA and ATAPI devices
device		ata
device		atadisk		# ATA disk drives
device		atapicd		# ATAPI CDROM drives
options 	ATA_STATIC_ID	# Static device numbering

# SCSI Controllers
device		bt		# Buslogic/Mylex MultiMaster SCSI adapters

# SCSI peripherals
device		scbus		# SCSI bus (required for SCSI)
device		ch		# SCSI media changers
device		da		# Direct Access (disks)
device		sa		# Sequential Access (tape etc)
device		cd		# CD
device		pass		# Passthrough device (direct SCSI access)
device		ses		# SCSI Environmental Services (and SAF-TE)

# atkbdc0 controls both the keyboard and the PS/2 mouse
device		atkbdc		# AT keyboard controller
device		atkbd		# AT keyboard
device		psm		# PS/2 mouse

device		vga		# VGA video card driver

device		splash		# Splash screen and screen saver support

# syscons is the default console driver, resembling an SCO console
device		sc

# Enable this for the pcvt (VT220 compatible) console driver
#device		vt
#options 	XSERVER		# support for X server on a vt console
#options 	FAT_CURSOR	# start with block cursor

device		agp		# support several AGP chipsets

# Power management support (see NOTES for more options)
#device		apm
# Add suspend/resume support for the i8254.
device		pmtimer

# PCCARD (PCMCIA) support
# PCMCIA and cardbus bridge support
device		cbb		# cardbus (yenta) bridge
device		pccard		# PC Card (16-bit) bus
device		cardbus		# CardBus (32-bit) bus

# Serial (COM) ports
device		sio		# 8250, 16[45]50 based serial ports
device		uart		# Generic UART driver

device		le		# NE2100, NE32-VL Lance Ethernet cards

# Pseudo devices.
device		loop		# Network loopback
device		random		# Entropy device
device		ether		# Ethernet support
device		sl		# Kernel SLIP
device		ppp		# Kernel PPP
device		tun		# Packet tunnel.
device		pty		# Pseudo-ttys (telnet etc)
device		md		# Memory "disks"
device		gif		# IPv6 and IPv4 tunneling
device		faith		# IPv6-to-IPv4 relaying (translation)

# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device		bpf		# Berkeley packet filter

# Crypto device required when you're using IPSEC
device		crypto		# Crypto devices and basic algorithms