5.1-BETA panic: bremfree: removing a buffer not on a queue

5.1-BETA from last weekend:

$ uname -a
FreeBSD pav.oook.cz 5.1-BETA FreeBSD 5.1-BETA #0: Sun May 11 13:45:37 CEST 2003     root_at_pav.oook.cz:/usr/obj/usr/src/sys/PAV  i386

I was listening to mp3 in xmms, usual desktop programs did their usual
background work. 


Fatal trap 12: page fault while in kernel mode
fault virtual address   = 0x24
fault code              = supervisor read, page not present
instruction pointer     = 0x8:0xc020dbbb
stack pointer           = 0x10:0xcd28ec44
frame pointer           = 0x10:0xcd28ec58
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 28 (irq5: uhci2 ehci0+)
trap number             = 12
panic: page fault

syncing disks, buffers remaining... panic: bremfree: removing a buffer not on a queue
Uptime: 6d6h53m9s


IRQ5 consumers:

uhci2: <VIA 83C572 USB controller> port 0xe800-0xe81f irq 5 at device 16.2 on pci0
usb2: <VIA 83C572 USB controller> on uhci2
usb2: USB revision 1.0
uhub2: VIA UHCI root hub, class 9/0, rev 1.00/1.00, addr 1
uhub2: 2 ports with 2 removable, self powered
ehci0: <EHCI (generic) USB 2.0 controller> mem 0xdfffff00-0xdfffffff irq 5 at device 16.3 on pci0
ehci_pci_attach: companion usb0
ehci_pci_attach: companion usb1
ehci_pci_attach: companion usb2
usb3: EHCI version 1.0
usb3: companion controllers, 2 ports each: usb0 usb1 usb2
usb3: <EHCI (generic) USB 2.0 controller> on ehci0
usb3: USB revision 2.0
uhub3: (0x1106) EHCI root hub, class 9/0, rev 2.00/1.00, addr 1
uhub3: 6 ports with 6 removable, self powered


(kgdb) bt full
#0  doadump () at /usr/src/sys/kern/kern_shutdown.c:238
No locals.
#1  0xc0216478 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:370
No locals.
#2  0xc021676b in panic () at /usr/src/sys/kern/kern_shutdown.c:543
        td = (struct thread *) 0xc263f000
        bootopt = 260
        newpanic = 0
        buf = "bremfree: removing a buffer not on a queue", '\0' <repeats 213 times>
#3  0xc0254970 in bremfreel (bp=0xc77a26b8) at /usr/src/sys/kern/vfs_bio.c:648
        old_qindex = 0
#4  0xc02548a5 in bremfree (bp=0x0) at /usr/src/sys/kern/vfs_bio.c:630
No locals.
#5  0xc0256b68 in vfs_bio_awrite (bp=0x0) at /usr/src/sys/kern/vfs_bio.c:1701
        i = -1033637888
        j = 0
        lblkno = 102528
        vp = (struct vnode *) 0xc271436c
        ncl = -948296008
        nwritten = -948296008
        size = -1071259275
        maxcl = -707564468
#6  0xc025dcce in vop_stdfsync (ap=0xd5d36cd0) at /usr/src/sys/kern/vfs_default.c:759
        vp = (struct vnode *) 0x0
        bp = (struct buf *) 0xc77a26b8
        nbp = (struct buf *) 0xc271436c
        error = 0
        maxretry = 100
#7  0xc01e1310 in spec_fsync (ap=0xd5d36cd0) at /usr/src/sys/fs/specfs/spec_vnops.c:418
No locals.
#8  0xc01e09a8 in spec_vnoperate (ap=0x0) at /usr/src/sys/fs/specfs/spec_vnops.c:123
No locals.
#9  0xc0265f6e in sched_sync () at vnode_if.h:612
        slp = (struct synclist *) 0xc261d788
        vp = (struct vnode *) 0xd5d36cd0
        mp = (struct mount *) 0xc26ec200
        starttime = 1053199229
        td = (struct thread *) 0xc263f000
#10 0xc020379e in fork_exit (callout=0xc0265df0 <sched_sync>, arg=0x0, frame=0x0) at /usr/src/sys/kern/kern_fork.c:792
        td = (struct thread *) 0x0
        p = (struct proc *) 0xc26d2b40


(kgdb) up 3
#3  0xc0254970 in bremfreel (bp=0xc77a26b8) at /usr/src/sys/kern/vfs_bio.c:648
648                             panic("bremfree: removing a buffer not on a queue");
(kgdb) list
643                     KASSERT(BUF_REFCNT(bp) == 1, ("bremfree: bp %p not locked",bp));
644                     TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
645                     bp->b_qindex = QUEUE_NONE;
646             } else {
647                     if (BUF_REFCNT(bp) <= 1)
648                             panic("bremfree: removing a buffer not on a queue");
649             }
650     
651             /*
652              * Fixup numfreebuffers count.  If the buffer is invalid or not

(kgdb) up
#4  0xc02548a5 in bremfree (bp=0x0) at /usr/src/sys/kern/vfs_bio.c:630
630             bremfreel(bp);
(kgdb) list
625      */
626     void
627     bremfree(struct buf * bp)
628     {
629             mtx_lock(&bqlock);
630             bremfreel(bp);
631             mtx_unlock(&bqlock);
632     }
633     
634     void

(kgdb) up
#5  0xc0256b68 in vfs_bio_awrite (bp=0x0) at /usr/src/sys/kern/vfs_bio.c:1701
1701            bremfree(bp);
(kgdb) list
1696                            splx(s);
1697                            return nwritten;
1698                    }
1699            }
1700    
1701            bremfree(bp);
1702            bp->b_flags |= B_ASYNC;
1703    
1704            splx(s);
1705            /*

(kgdb) up
#6  0xc025dcce in vop_stdfsync (ap=0xd5d36cd0) at /usr/src/sys/kern/vfs_default.c:759
759                             vfs_bio_awrite(bp);
(kgdb) list
754                             continue;
755                     VI_UNLOCK(vp);
756                     if ((bp->b_flags & B_DELWRI) == 0)
757                             panic("fsync: not dirty");
758                     if ((vp->v_vflag & VV_OBJBUF) && (bp->b_flags & B_CLUSTEROK)) {
759                             vfs_bio_awrite(bp);
760                             splx(s);
761                     } else {
762                             bremfree(bp);
763                             splx(s);

I have vmcore and debug kernel handy.

-- 
Pav Lucistnik <pav_at_oook.cz>
Ako rozoznate skuseneho hackera od zaciatocnika?
Zaciatocnik si mysli, ze kilobyte ma 1000 byte-ov a skuseny si mysli, ze
kilometer ma 1024 metrov.