Re: -CURRENT panics on intensive fs operations.

On Tue, 2007-02-20 at 09:48 -0500, Alexandre "Sunny" Kovalenko wrote:
> On Wed, 2007-02-14 at 20:14 -0500, Kris Kennaway wrote:
> > On Wed, Feb 14, 2007 at 07:48:51PM -0500, Alexandre Sunny Kovalenko wrote:
> > > On Tue, 2007-02-13 at 20:17 -0500, Kris Kennaway wrote:
> > > > On Tue, Feb 13, 2007 at 08:02:39PM -0500, Alexandre Sunny Kovalenko wrote:
> > > > > I can reliably panic -CURRENT (Feb 11, noon EST) with the something that
> > > > > excersises the file system. I have currently settled on (cd /usr/ports;
> > > > > make clean), but it all started out as doing some "emerges" to test the
> > > > > latest linuxolator. In the case of the "make clean" I have seen it
> > > > > crashing as early as /usr/ports/audio and as late
> > > > > as /usr/ports/textproc. 
> > > > > 
I am still not capable to get good backtrace from the kernel dump, but I
have managed to hook up remote console to this machine, so here are
results:

db> bt
Tracing pid 33 tid 100032 td 0xc4cee510
kdb_enter(c067c69d) at kdb_enter+0x2b
panic(c0667ba3,c306d5c0,c306d5c0,e38a2cfc,c0619fd9,...) at panic+0x11c
vm_pageq_remove_nowakeup(c306d5c0,c061a0b8,e38a2d04,c061a0ee,e38a2d24,...) at vm_pageq_remove_nowakeup+0x35
vm_page_zero_idle(e38a2d24,c04c7fe4,0,e38a2d38,c4ef8900,...) at
vm_page_zero_idle+0x49
vm_pagezero(0,e38a2d38) at vm_pagezero+0x36
fork_exit(c061a0b8,0,e38a2d38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xe38a2d70, ebp = 0 ---
db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
<snip>
   33     0     0     0  RL      CPU 0               [pagezero]
<snip>

... and (hopefully) relevant bits from the source
(kgdb) list *vm_pageq_remove_nowakeup+0x35
0xc06192f9 is in vm_pageq_remove_nowakeup
(/usr/src/sys/vm/vm_pageq.c:223).
218             struct vpgqueues *pq;
219
220             if (queue != PQ_NONE) {
221                     pq = &vm_page_queues[queue];
222                     VM_PAGE_SETQUEUE2(m, PQ_NONE);
223                     TAILQ_REMOVE(&pq->pl, m, pageq);
224                     (*pq->cnt)--;
225                     pq->lcnt--;
226             }
227     }
(kgdb) list *vm_page_zero_idle+0x49
0xc0619fd9 is in vm_page_zero_idle (/usr/src/sys/vm/vm_zeroidle.c:111).
106             mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
107             zero_state = 0;
108             m = vm_pageq_find(PQ_FREE, free_rover, FALSE);
109             if (m != NULL && (m->flags & PG_ZERO) == 0) {
110                     vm_pageq_remove_nowakeup(m);
111                     mtx_unlock(&vm_page_queue_free_mtx);
112                     pmap_zero_page_idle(m);
113                     mtx_lock(&vm_page_queue_free_mtx);
114                     m->flags |= PG_ZERO;
115                     vm_pageq_enqueue(PQ_FREE + m->pc, m);
(kgdb) list *vm_pagezero+0x36
0xc061a0ee is in vm_pagezero (/usr/src/sys/vm/vm_zeroidle.c:147).
142             mtx_lock(&vm_page_queue_free_mtx);
143             for (;;) {
144                     if (vm_page_zero_check()) {
145                             vm_page_zero_idle();
146     #ifndef PREEMPTION
147                             if (sched_runnable()) {
148                                     mtx_lock_spin(&sched_lock);
149                                     mi_switch(SW_VOL, NULL);
150                                     mtx_unlock_spin(&sched_lock);
151                             }
(kgdb)

This is -CURRENT from Feb. 21 20:00 EST, configuration file attached.

As I have stated previously, I can reliably reproduce this given enough
time -- worst I have seen is about 18 hours of (cd /usr/ports; make
clean). I do have remote debug configured on this machine, so I can do
more digging if necessary, but at this point I am way out of my depth
with this VM stuff.

-- 
Alexandre "Sunny" Kovalenko

#
# TPX60 -- kernel configuration file for ThinkPad X60
#

cpu		I686_CPU
ident		TPX60

# To statically compile in device wiring instead of /boot/device.hints
#hints		"GENERIC.hints"		# Default places to look for devices.

makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols

options 	SCHED_4BSD		# 4BSD scheduler
# options 	PREEMPTION		# Enable kernel thread preemption
options 	INET			# InterNETworking
options 	INET6			# IPv6 communications protocols
options 	FFS			# Berkeley Fast Filesystem
options 	SOFTUPDATES		# Enable FFS soft updates support
options 	UFS_ACL			# Support for access control lists
options 	UFS_DIRHASH		# Improve performance on big directories
options 	NFSCLIENT		# Network Filesystem Client
options 	NFSSERVER		# Network Filesystem Server
options 	MSDOSFS			# MSDOS Filesystem
options 	CD9660			# ISO 9660 Filesystem
options 	PROCFS			# Process filesystem (requires PSEUDOFS)
options 	PSEUDOFS		# Pseudo-filesystem framework
options 	GEOM_PART_GPT		# GUID Partition Tables.
options 	GEOM_LABEL		# Provides labelization
options 	COMPAT_43TTY		# BSD 4.3 TTY compat [KEEP THIS!]
options 	COMPAT_FREEBSD4		# Compatible with FreeBSD4
options 	COMPAT_FREEBSD5		# Compatible with FreeBSD5
options 	COMPAT_FREEBSD6		# Compatible with FreeBSD6
options 	SCSI_DELAY=5000		# Delay (in ms) before probing SCSI
options 	KTRACE			# ktrace(1) support
options 	SYSVSHM			# SYSV-style shared memory
options 	SYSVMSG			# SYSV-style message queues
options 	SYSVSEM			# SYSV-style semaphores
options 	_KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options 	KBD_INSTALL_CDEV	# install a CDEV entry in /dev
options 	ADAPTIVE_GIANT		# Giant mutex is adaptive.
options 	STOP_NMI		# Stop CPUS using NMI instead of IPI

# Debugging for use in -current
options 	KDB			# Enable kernel debugger support.
options 	DDB			# Support DDB.
options 	GDB			# Support remote GDB.
options 	INVARIANTS		# Enable calls of extra sanity checking
options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
options 	WITNESS			# Enable checks to detect deadlocks and cycles
# options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed

# To make an SMP kernel, the next two lines are needed
options 	SMP			# Symmetric MultiProcessor Kernel
device		apic			# I/O APIC

# Bus support.
device		eisa
device		pci

# ATA and ATAPI devices
device		ata
device		atadisk		# ATA disk drives
options 	ATA_STATIC_ID	# Static device numbering

# SCSI peripherals
device		scbus		# SCSI bus (required for SCSI)
device		da		# Direct Access (disks)
device		cd		# CD
device		pass		# Passthrough device (direct SCSI access)

# atkbdc0 controls both the keyboard and the PS/2 mouse
device		atkbdc		# AT keyboard controller
device		atkbd		# AT keyboard
device		psm		# PS/2 mouse

device		kbdmux		# keyboard multiplexer

device		vga		# VGA video card driver

device		splash		# Splash screen and screen saver support

# syscons is the default console driver, resembling an SCO console
device		sc

device		agp		# support several AGP chipsets

# Add suspend/resume support for the i8254.
device		pmtimer

# PCCARD (PCMCIA) support
# PCMCIA and cardbus bridge support
device		cbb		# cardbus (yenta) bridge
device		pccard		# PC Card (16-bit) bus
device		cardbus		# CardBus (32-bit) bus

# Pseudo devices.
device		loop		# Network loopback
device		random		# Entropy device
device		ether		# Ethernet support
device		ppp		# Kernel PPP
device		tun		# Packet tunnel.
device		pty		# Pseudo-ttys (telnet etc)
device		md		# Memory "disks"
device		gif		# IPv6 and IPv4 tunneling
device		faith		# IPv6-to-IPv4 relaying (translation)
device		firmware	# firmware assist module

# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device		bpf		# Berkeley packet filter

# FireWire support
# device		firewire	# FireWire bus code
# device		dcons_crom
# device 		dcons