We are having regular panics on several machines in the cluster. Below follows the script from the kgdb(1) session, hopefully providing enough information. This machine runs 11.0-CURRENT #2 r262892, from 2 days ago. It uses tmpfs(5) for the port build workspace. I have an unconfirmed suspicion that use of sysutils/lsof is involved somehow, but cannot be sure. (In my experience with panics with port building, removing lsof from the system did have an effect, but I may be going down the wrong rabbit hole.) Script started on Sun Mar 9 16:40:07 2014 root_at_redbuild01.nyi:/usr/obj/usr/src/sys/REDBUILD # sh # kgdb ./kernel.debug /var/crash/vmcore.1 GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd"... Unread portion of the kernel message buffer: panic: vm_fault: fault on nofault entry, addr: fffffe035021a000 cpuid = 1 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe1839a54180 kdb_backtrace() at kdb_backtrace+0x39/frame 0xfffffe1839a54230 panic() at panic+0x155/frame 0xfffffe1839a542b0 vm_fault_hold() at vm_fault_hold+0x1e7a/frame 0xfffffe1839a54500 vm_fault() at vm_fault+0x77/frame 0xfffffe1839a54540 trap_pfault() at trap_pfault+0x199/frame 0xfffffe1839a545e0 trap() at trap+0x4a0/frame 0xfffffe1839a547f0 calltrap() at calltrap+0x8/frame 0xfffffe1839a547f0 --- trap 0xc, rip = 0xffffffff80d97bab, rsp = 0xfffffe1839a548b0, rbp = 0xfffffe1839a54910 --- copyout() at copyout+0x3b/frame 0xfffffe1839a54910 memrw() at memrw+0x19f/frame 0xfffffe1839a54950 giant_read() at giant_read+0xa4/frame 0xfffffe1839a54990 devfs_read_f() at devfs_read_f+0xeb/frame 0xfffffe1839a549f0 dofileread() at dofileread+0x95/frame 0xfffffe1839a54a40 kern_readv() at kern_readv+0x68/frame 0xfffffe1839a54a90 sys_read() at sys_read+0x63/frame 0xfffffe1839a54ae0 amd64_syscall() at amd64_syscall+0x3fb/frame 0xfffffe1839a54bf0 Xfast_syscall() at Xfast_syscall+0xfb/frame 0xfffffe1839a54bf0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800b8444a, rsp = 0x7fffffffd088, rbp = 0x7fffffffd0d0 --- KDB: enter: panic Reading symbols from /boot/kernel/zfs.ko.symbols...done. Loaded symbols for /boot/kernel/zfs.ko.symbols Reading symbols from /boot/kernel/opensolaris.ko.symbols...done. Loaded symbols for /boot/kernel/opensolaris.ko.symbols Reading symbols from /boot/kernel/ums.ko.symbols...done. Loaded symbols for /boot/kernel/ums.ko.symbols Reading symbols from /boot/kernel/tmpfs.ko.symbols...done. Loaded symbols for /boot/kernel/tmpfs.ko.symbols Reading symbols from /boot/kernel/nullfs.ko.symbols...done. Loaded symbols for /boot/kernel/nullfs.ko.symbols Reading symbols from /boot/kernel/linprocfs.ko.symbols...done. Loaded symbols for /boot/kernel/linprocfs.ko.symbols Reading symbols from /boot/kernel/linux.ko.symbols...done. Loaded symbols for /boot/kernel/linux.ko.symbols #0 doadump (textdump=-967130448) at pcpu.h:219 219 __asm("movq %%gs:%1,%0" : "=r" (td) (kgdb) bt #0 doadump (textdump=-967130448) at pcpu.h:219 #1 0xffffffff8034a1a5 in db_fncall (dummy1=<value optimized out>, dummy2=<value optimized out>, dummy3=<value optimized out>, dummy4=<value optimized out>) at /usr/src/sys/ddb/db_command.c:578 #2 0xffffffff80349e8d in db_command (cmd_table=0x0) at /usr/src/sys/ddb/db_command.c:449 #3 0xffffffff80349c04 in db_command_loop () at /usr/src/sys/ddb/db_command.c:502 #4 0xffffffff8034c660 in db_trap (type=<value optimized out>, code=0) at /usr/src/sys/ddb/db_main.c:231 #5 0xffffffff80987ae9 in kdb_trap (type=3, code=0, tf=<value optimized out>) at /usr/src/sys/kern/subr_kdb.c:656 #6 0xffffffff80d999b9 in trap (frame=0xfffffe1839a54160) at /usr/src/sys/amd64/amd64/trap.c:571 #7 0xffffffff80d7e6e2 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:231 #8 0xffffffff8098724e in kdb_enter (why=0xffffffff8100f4ba "panic", msg=<value optimized out>) at cpufunc.h:63 #9 0xffffffff80946a75 in panic (fmt=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:752 #10 0xffffffff80c0a1fa in vm_fault_hold (map=<value optimized out>, vaddr=<value optimized out>, fault_type=<value optimized out>, fault_flags=<value optimized out>, m_hold=<value optimized out>) at /usr/src/sys/vm/vm_fault.c:272 #11 0xffffffff80c08337 in vm_fault (map=0xfffff80002000000, vaddr=<value optimized out>, fault_type=1 '\001', fault_flags=128) at /usr/src/sys/vm/vm_fault.c:217 #12 0xffffffff80d9a1a9 in trap_pfault (frame=0xfffffe1839a54800, usermode=0) at /usr/src/sys/amd64/amd64/trap.c:767 #13 0xffffffff80d999d0 in trap (frame=0xfffffe1839a54800) at /usr/src/sys/amd64/amd64/trap.c:455 #14 0xffffffff80d7e6e2 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:231 #15 0xffffffff80d97bab in copyout () at /usr/src/sys/amd64/amd64/support.S:246 #16 0xffffffff8099c2f5 in uiomove_faultflag (cp=<value optimized out>, n=<value optimized out>, uio=0xfffffe1839a54ab0, nofault=<value optimized out>) at /usr/src/sys/kern/subr_uio.c:192 #17 0xffffffff80d8612f in memrw (dev=0xfffff8000dbd0400, uio=0xfffffe1839a54ab0, flags=113246208) at /usr/src/sys/amd64/amd64/mem.c:101 #18 0xffffffff808ecf04 in giant_read (dev=0xfffff8000dbd0400, uio=0xfffffe1839a54ab0, ioflag=0) at /usr/src/sys/kern/kern_conf.c:442 #19 0xffffffff808185cb in devfs_read_f (fp=0xfffff80083439230, uio=0xfffffe1839a54ab0, cred=<value optimized out>, flags=0, td=0xfffff80e4edb8490) at /usr/src/sys/fs/devfs/devfs_vnops.c:1193 #20 0xffffffff809a15e5 in dofileread (td=0xfffff80e4edb8490, fd=4, fp=0xfffff80083439230, auio=0xfffffe1839a54ab0, offset=<value optimized out>, flags=1172307968) at file.h:299 #21 0xffffffff809a1308 in kern_readv (td=0xfffff80e4edb8490, fd=4, auio=0xfffffe1839a54ab0) at /usr/src/sys/kern/sys_generic.c:256 #22 0xffffffff809a1293 in sys_read (td=<value optimized out>, uap=<value optimized out>) at /usr/src/sys/kern/sys_generic.c:171 #23 0xffffffff80d9a9fb in amd64_syscall (td=0xfffff80e4edb8490, traced=0) at subr_syscall.c:133 #24 0xffffffff80d7e9cb in Xfast_syscall () at /usr/src/sys/amd64/amd64/exception.S:390 #25 0x0000000800b8444a in ?? () Previous frame inner to this frame (corrupt stack?) Current language: auto; currently minimal (kgdb) frame 19 #19 0xffffffff808185cb in devfs_read_f (fp=0xfffff80083439230, uio=0xfffffe1839a54ab0, cred=<value optimized out>, flags=0, td=0xfffff80e4edb8490) at /usr/src/sys/fs/devfs/devfs_vnops.c:1193 1193 error = dsw->d_read(dev, uio, ioflag); (kgdb) list 1188 ioflag = fp->f_flag & (O_NONBLOCK | O_DIRECT); 1189 if (ioflag & O_DIRECT) 1190 ioflag |= IO_DIRECT; 1191 1192 foffset_lock_uio(fp, uio, flags | FOF_NOLOCK); 1193 error = dsw->d_read(dev, uio, ioflag); 1194 if (uio->uio_resid != resid || (error == 0 && resid != 0)) 1195 vfs_timestamp(&dev->si_atime); 1196 td->td_fpop = fpop; 1197 dev_relthread(dev, ref); (kgdb) down #18 0xffffffff808ecf04 in giant_read (dev=0xfffff8000dbd0400, uio=0xfffffe1839a54ab0, ioflag=0) at /usr/src/sys/kern/kern_conf.c:442 442 retval = dsw->d_gianttrick->d_read(dev, uio, ioflag); (kgdb) list 437 438 dsw = dev_refthread(dev, &ref); 439 if (dsw == NULL) 440 return (ENXIO); 441 mtx_lock(&Giant); 442 retval = dsw->d_gianttrick->d_read(dev, uio, ioflag); 443 mtx_unlock(&Giant); 444 dev_relthread(dev, ref); 445 return (retval); 446 } (kgdb) p *dev $1 = {si_spare0 = 0x0, si_flags = 4, si_atime = {tv_sec = 1394286776, tv_nsec = 0}, si_ctime = {tv_sec = 1394236183, tv_nsec = 584945000}, si_mtime = {tv_sec = 1394236183, tv_nsec = 584945000}, si_uid = 0, si_gid = 2, si_mode = 416, si_cred = 0x0, si_drv0 = 1, si_refcount = 9, si_list = {le_next = 0xfffff8000dbd0600, le_prev = 0xffffffff8144db18}, si_clone = {le_next = 0x0, le_prev = 0x0}, si_children = {lh_first = 0x0}, si_siblings = { le_next = 0x0, le_prev = 0x0}, si_parent = 0x0, si_mountpt = 0x0, si_drv1 = 0x0, si_drv2 = 0x0, si_devsw = 0xffffffff8144da78, si_iosize_max = 65536, si_usecount = 1, si_threadcount = 2, __si_u = {__sid_snapdata = 0x0}, si_name = "kmem", '\0' <repeats 59 times>} (kgdb) p *uio $2 = {uio_iov = 0xfffffe1839a54aa0, uio_iovcnt = 1, uio_offset = -2184830705664, uio_resid = 113246208, uio_segflg = UIO_USERSPACE, uio_rw = UIO_READ, uio_td = 0xfffff80e4edb8490} (kgdb) p *ioflag Cannot access memory at address 0x0 (kgdb) p Giant $3 = {lock_object = {lo_name = 0xffffffff8100e05a "Giant", lo_flags = 17498112, lo_data = 0, lo_witness = 0x0}, mtx_lock = 18446735339069080720} (kgdb) down #17 0xffffffff80d8612f in memrw (dev=0xfffff8000dbd0400, uio=0xfffffe1839a54ab0, flags=113246208) at /usr/src/sys/amd64/amd64/mem.c:101 101 error = uiomove((void *)PHYS_TO_DMAP(v), (int)c, uio); (kgdb) list 96 if (dev2unit(dev) == CDEV_MINOR_MEM) { 97 v = uio->uio_offset; 98 kmemphys: 99 o = v & PAGE_MASK; 100 c = min(uio->uio_resid, (u_int)(PAGE_SIZE - o)); 101 error = uiomove((void *)PHYS_TO_DMAP(v), (int)c, uio); 102 continue; 103 } 104 else if (dev2unit(dev) == CDEV_MINOR_KMEM) { 105 v = uio->uio_offset; (kgdb) p *v $4 = 0 (kgdb) p *c $5 = 0 (kgdb) p *uio $6 = {uio_iov = 0xfffffe1839a54aa0, uio_iovcnt = 1, uio_offset = -2184830705664, uio_resid = 113246208, uio_segflg = UIO_USERSPACE, uio_rw = UIO_READ, uio_td = 0xfffff80e4edb8490} (kgdb) down #16 0xffffffff8099c2f5 in uiomove_faultflag (cp=<value optimized out>, n=<value optimized out>, uio=0xfffffe1839a54ab0, nofault=<value optimized out>) at /usr/src/sys/kern/subr_uio.c:192 192 error = copyout(cp, iov->iov_base, cnt); (kgdb) list 187 switch (uio->uio_segflg) { 188 189 case UIO_USERSPACE: 190 maybe_yield(); 191 if (uio->uio_rw == UIO_READ) 192 error = copyout(cp, iov->iov_base, cnt); 193 else 194 error = copyin(iov->iov_base, cp, cnt); 195 if (error) 196 goto out; (kgdb) p *cp Attempt to dereference a generic pointer. (kgdb) p cp $7 = <value optimized out> (kgdb) down #15 0xffffffff80d97bab in copyout () at /usr/src/sys/amd64/amd64/support.S:246 246 cld Current language: auto; currently asm (kgdb) list 241 xchgq %rdi,%rsi 242 /* bcopy(%rsi, %rdi, %rdx) */ 243 movq %rdx,%rcx 244 245 shrq $3,%rcx 246 cld 247 rep 248 movsq 249 movb %dl,%cl 250 andb $7,%cl (kgdb) down #14 0xffffffff80d7e6e2 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:231 231 call trap (kgdb) list 226 #endif 227 .globl calltrap 228 .type calltrap,_at_function 229 calltrap: 230 movq %rsp,%rdi 231 call trap 232 MEXITCOUNT 233 jmp doreti /* Handle any pending ASTs */ 234 235 /* (kgdb) quit Script done on Sun Mar 9 16:46:04 2014 Glen
This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:40:47 UTC