Re: 8.0-RC1: kernel page fault in NLM master thread (VIMAGE or ZFS related?)

From: Jamie Gritton <jamie_at_FreeBSD.org>
Date: Fri, 25 Sep 2009 17:01:13 -0600
Marcel Moolenaar wrote:
> All,
> 
> I just got this overnight on my server:
> 
> Fatal trap 12: page fault while in kernel mode
> fault virtual address    = 0x90
> fault code        = supervisor read, page not present
> instruction pointer    = 0x20:0xc05ba39d
> stack pointer            = 0x28:0xf31077bc
> frame pointer            = 0x28:0xf31077c8
> code segment        = base 0x0, limit 0xfffff, type 0x1b
>             = DPL 0, pres 1, def32 1, gran 1
> processor eflags    = interrupt enabled, resume, IOPL = 0
> current process        = 928 (NLM: master)
> 
> (kgdb) bt
> #0  doadump () at pcpu.h:246
> #1  0xc05e03f3 in boot (howto=260) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_shutdown.c:416
> #2  0xc05e062d in panic (fmt=Variable "fmt" is not available.
> ) at /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_shutdown.c:579
> #3  0xc04ac807 in db_panic (addr=Could not find the frame base for 
> "db_panic".
> ) at /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_command.c:478
> #4  0xc04acd91 in db_command (last_cmdp=0xc0881c3c, cmd_table=0x0, 
> dopager=1) at /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_command.c:445
> #5  0xc04aceea in db_command_loop () at 
> /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_command.c:498
> #6  0xc04aed5d in db_trap (type=12, code=0) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/ddb/db_main.c:229
> #7  0xc0608a14 in kdb_trap (type=12, code=0, tf=0xf310777c) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/kern/subr_kdb.c:535
> #8  0xc07c53af in trap_fatal (frame=0xf310777c, eva=144) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:924
> #9  0xc07c5650 in trap_pfault (frame=0xf310777c, usermode=0, eva=144) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:846
> #10 0xc07c5ff2 in trap (frame=0xf310777c) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:528
> #11 0xc07ac50b in calltrap () at 
> /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/exception.s:165
> #12 0xc05ba39d in prison_priv_check (cred=0xc61e4880, priv=334) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_jail.c:3568
> #13 0xc05d39ee in priv_check_cred (cred=0xc61e4880, priv=334, flags=0) 
> at /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_priv.c:92
> #14 0xc09dbffc in secpolicy_fs_owner (mp=0xc4112284, cred=0xc61e4880) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/compat/opensolaris/kern/opensolaris_policy.c:86 
> 
> #15 0xc09dc527 in secpolicy_vnode_access (cred=0xc61e4880, 
> vp=0xc4bb6d9c, owner=501, accmode=128)
>     at 
> /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/compat/opensolaris/kern/opensolaris_policy.c:125 
> 
> #16 0xc0a56c5c in zfs_zaccess (zp=0xd4be8658, mode=2, flags=Variable 
> "flags" is not available.
> ) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c:2445 
> 
> #17 0xc0a56edb in zfs_zaccess_rwx (zp=0xd4be8658, mode=Variable "mode" 
> is not available.
> ) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c:2484 
> 
> #18 0xc0a6bfa4 in zfs_freebsd_access (ap=0xf31078d4) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:1068 
> 
> #19 0xc07cfeb2 in VOP_ACCESS_APV (vop=0xc0acfac0, a=0xf31078d4) at 
> vnode_if.c:571
> #20 0xc0718c93 in nlm_get_vfs_state (host=Variable "host" is not available.
> ) at vnode_if.h:254
> #21 0xc0718e30 in nlm_do_unlock (argp=0xf31079c8, result=0xf3107a08, 
> rqstp=0xcb199800, rpcp=0x0) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_impl.c:2227
> #22 0xc071ac87 in nlm4_unlock_4_svc (argp=0xf31079c8, result=0xf3107a08, 
> rqstp=0xcb199800) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_server.c:540
> #23 0xc071bce3 in nlm_prog_4 (rqstp=0xcb199800, transp=0xc652de00) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_svc.c:512
> #24 0xc07284bf in svc_run_internal (pool=0xc61e4c80, ismaster=1) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/rpc/svc.c:893
> #25 0xc072943d in svc_run (pool=0xc61e4c80) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/rpc/svc.c:1233
> #26 0xc071a348 in nlm_syscall (td=0xc6551000, uap=0xf3107cf8) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/nlm/nlm_prot_impl.c:1593
> #27 0xc07c5977 in syscall (frame=0xf3107d38) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/trap.c:1073
> #28 0xc07ac570 in Xint0x80_syscall () at 
> /zmirror/nfs/freebsd/base/stable/8/sys/i386/i386/exception.s:261
> #29 0x00000033 in ?? ()
> 
> (kgdb) frame 12
> #12 0xc05ba39d in prison_priv_check (cred=0xc61e4880, priv=334) at 
> /zmirror/nfs/freebsd/base/stable/8/sys/kern/kern_jail.c:3568
> 3568        switch (priv) {
> (kgdb) l 3567
> 3562         */
> 3563            if (cred->cr_prison->pr_flags & PR_VNET)
> 3564                return (0);
> 3565        }
> 3566    #endif /* VIMAGE */
> 3567   
> 3568        switch (priv) {
> 3569   
> 3570            /*
> 3571             * Allow ktrace privileges for root in jail.
> (kgdb) p cred->cr_prison
> $4 = (struct prison *) 0x0

It seems to be NFS related.  I think the null pointer in question is
from the export's anonymous credential.  Try the patch below and see
if it helps (which I guess means run it overnight and see if it
crashes again).  I've also patched a similar missing cred prison in
GSS_SVC, since I'm not versed enough in NFS/RPC stuff to know if it
might be the problem.

- Jamie


Index: kern/vfs_export.c
===================================================================
--- kern/vfs_export.c	(revision 197506)
+++ kern/vfs_export.c	(working copy)
_at__at_ -122,6 +122,8 _at__at_
  		np->netc_anon->cr_uid = argp->ex_anon.cr_uid;
  		crsetgroups(np->netc_anon, argp->ex_anon.cr_ngroups,
  		    argp->ex_anon.cr_groups);
+		np->netc_anon->cr_prison = &prison0;
+		prison_hold(np->netc_anon->cr_prison);
  		np->netc_numsecflavors = argp->ex_numsecflavors;
  		bcopy(argp->ex_secflavors, np->netc_secflavors,
  		    sizeof(np->netc_secflavors));
_at__at_ -206,6 +208,8 _at__at_
  	np->netc_anon->cr_uid = argp->ex_anon.cr_uid;
  	crsetgroups(np->netc_anon, argp->ex_anon.cr_ngroups,
  	    np->netc_anon->cr_groups);
+	np->netc_anon->cr_prison = &prison0;
+	prison_hold(np->netc_anon->cr_prison);
  	np->netc_numsecflavors = argp->ex_numsecflavors;
  	bcopy(argp->ex_secflavors, np->netc_secflavors,
  	    sizeof(np->netc_secflavors));
Index: rpc/rpcsec_gss/svc_rpcsec_gss.c
===================================================================
--- rpc/rpcsec_gss/svc_rpcsec_gss.c	(revision 197506)
+++ rpc/rpcsec_gss/svc_rpcsec_gss.c	(working copy)
_at__at_ -449,6 +449,8 _at__at_
  	cr->cr_uid = cr->cr_ruid = cr->cr_svuid = uc->uid;
  	cr->cr_rgid = cr->cr_svgid = uc->gid;
  	crsetgroups(cr, uc->gidlen, uc->gidlist);
+	cr->cr_prison = &prison0;
+	prison_hold(cr->cr_prison);
  	*crp = crhold(cr);

  	return (TRUE);
Received on Fri Sep 25 2009 - 21:13:37 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:39:56 UTC