On 4 February 2013 16:06, Andriy Gapon <avg_at_freebsd.org> wrote: > on 04/02/2013 13:49 Sergey Kandaurov said the following: >> Hi, Rick! Here is the requested info regarding witness, and a bit more. >> The triggered KASSERT is now different though. > > It's exactly the same problem though :-) > Do you have a crashdump? > If yes, please print **vpp. Yep, but it's in a bad form :( It has many bits optimized out (compiled with clang). I'll rebuild the kernel with -O or so and will try again. #8 0xffffffff808bc4ce in kdb_enter (why=0xffffffff80e7ed99 "panic", msg=<value optimized out>) at cpufunc.h:63 #9 0xffffffff80888fb7 in vpanic (fmt=<value optimized out>, ap=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:746 #10 0xffffffff80888e66 in kassert_panic (fmt=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:641 #11 0xffffffff808d2259 in witness_checkorder (lock=0xfffffe00a1b44240, ---Type <return> to continue, or q <return> to quit--- flags=1, file=0xffffffff81b2bb36 "/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c", line=1452, interlock=<value optimized out>) at /usr/src/sys/kern/subr_witness.c:1122 #12 0xffffffff8086c11e in __lockmgr_args (lk=0xfffffe00a1b44240, flags=<value optimized out>, ilk=0xfffffe00a1b44270, wmesg=0xffffffff81b2808d "zfs", pri=96, timo=51, file=0xffffffff80e8e407 "/usr/src/sys/kern/vfs_default.c", line=0) at /usr/src/sys/kern/kern_lock.c:511 #13 0xffffffff8091439c in vop_stdlock (ap=<value optimized out>) at lockmgr.h:97 #14 0xffffffff80cb70c0 in VOP_LOCK1_APV (vop=<value optimized out>, a=<value optimized out>) at vnode_if.c:2022 #15 0xffffffff80932fbb in _vn_lock (vp=0xfffffe00a1b441d8, flags=<value optimized out>, file=0xffffffff81b2bb36 "/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c", line=1452) at vnode_if.h:859 #16 0xffffffff81abd902 in zfs_lookup () at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:1452 #17 0xffffffff81abdc1d in zfs_freebsd_lookup (ap=0xffffff848e6c0270) at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:5864 ---Type <return> to continue, or q <return> to quit--- #18 0xffffffff80cb51b2 in VOP_CACHEDLOOKUP_APV (vop=<value optimized out>, a=<value optimized out>) at vnode_if.c:191 #19 0xffffffff8091177f in vfs_cache_lookup (ap=<value optimized out>) at vnode_if.h:80 #20 0xffffffff80cb50a2 in VOP_LOOKUP_APV (vop=<value optimized out>, a=<value optimized out>) at vnode_if.c:125 #21 0xffffffff80919658 in lookup (ndp=0xffffff848e6c05d8) at vnode_if.h:54 #22 0xffffffff807dffe5 in nfsvno_namei (nd=0xffffff848e6c08c8, ndp=0xffffff848e6c05d8, dp=<value optimized out>, islocked=<value optimized out>, exp=<value optimized out>, p=<value optimized out>, retdirp=<value optimized out>) at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:413 #23 0xffffffff807d8ffa in nfsrvd_lookup (nd=0xffffff848e6c08c8, isdgram=<value optimized out>, dp=0xfffffe00a1b441d8, vpp=0xffffff848e6c0810, fhp=0xffffff848e6c07f4, p=0xfffffe00a1198000, exp=0xffffff848e6c07a0) at /usr/src/sys/fs/nfsserver/nfs_nfsdserv.c:517 #24 0xffffffff807cb845 in nfsrvd_dorpc (nd=0xffffff848e6c08c8, isdgram=0, p=0xfffffe00a1198000) at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:823 #25 0xffffffff807d7af2 in nfssvc_program (rqst=0xfffffe00a17bb000, xprt=0xfffffe00a124b200) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:347 #26 0xffffffff80a70659 in svc_run_internal (pool=0xfffffe00067db600, ismaster=0) at /usr/src/sys/rpc/svc.c:895 #27 0xffffffff80a7155b in svc_thread_start (arg=0xffffff848e6bfc50) ---Type <return> to continue, or q <return> to quit--- at /usr/src/sys/rpc/svc.c:1200 #28 0xffffffff80858944 in fork_exit ( callout=0xffffffff80a71550 <svc_thread_start>, arg=0xfffffe00067db600, frame=0xffffff848e6c0c00) at /usr/src/sys/kern/kern_fork.c:991 #29 0xffffffff80bfa86e in fork_trampoline () at exception.S:602 #30 0x0000000000000080 in ?? () #31 0x00007fffffffd820 in ?? () #32 0x0000000000000001 in ?? () #33 0x0000000000000000 in ?? () Current language: auto; currently minimal (kgdb) p vpp Cannot access memory at address 0x0 (kgdb) p dvp Cannot access memory at address 0x0 > >> Full witness is at http://people.freebsd.org/~pluknet/witness-zfs-20130204.txt >> >> shared lock of (lockmgr) zfs _at_ >> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:1452 >> while exclusively locked from >> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c:1747 >> panic: share->excl >> cpuid = 2 >> KDB: enter: panic >> [ thread pid 812 tid 100884 ] >> Stopped at kdb_enter+0x3e: movq $0,kdb_why >> >> The 1st line is at zfs_lookup(): >> if (error == 0 && (nm[0] != '.' || nm[1] != '\0')) { >> int ltype = 0; >> >> if (cnp->cn_flags & ISDOTDOT) { >> ltype = VOP_ISLOCKED(dvp); >> VOP_UNLOCK(dvp, 0); >> } >> ZFS_EXIT(zfsvfs); >> error = zfs_vnode_lock(*vpp, cnp->cn_lkflags); >> if (cnp->cn_flags & ISDOTDOT) >> ==> vn_lock(dvp, ltype | LK_RETRY); >> if (error != 0) { >> VN_RELE(*vpp); >> *vpp = NULL; >> return (error); >> } >> } else { >> ZFS_EXIT(zfsvfs); >> } >> >> The 2nd line is at zfs_vnode_lock(): >> int >> zfs_vnode_lock(vnode_t *vp, int flags) >> { >> int error; >> >> ASSERT(vp != NULL); >> >> error = vn_lock(vp, flags); >> return (error); >> } >> >> db> show locks >> exclusive lockmgr zfs (zfs) r = 0 (0xfffffe00a1b44240) locked _at_ >> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c:1747 >> db> show alllocks >> Process 812 (nfsd) thread 0xfffffe00a1198000 (100884) >> exclusive lockmgr zfs (zfs) r = 0 (0xfffffe00a1b44240) locked _at_ >> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c:1747 >> Process 750 (syslogd) thread 0xfffffe0015a4c480 (100706) >> exclusive lockmgr ufs (ufs) r = 0 (0xfffffe00a1962d50) locked _at_ >> /usr/src/sys/kern/vfs_syscalls.c:3433 >> Process 12 (intr) thread 0xfffffe0006813000 (100033) >> exclusive sleep mutex AAC I/O lock (AAC I/O lock) r = 0 >> (0xffffff8001bfb210) locked _at_ /usr/src/sys/dev/aac/aac.c:827 >> >> db> show lock 0xfffffe00a1b44240 >> class: lockmgr >> name: zfs >> state: XLOCK: 0xfffffe00a1198000 (tid 100884, pid 812, "nfsd") >> waiters: none >> spinners: none >> >> As KASSERT is different: >> >> db> bt >> Tracing pid 812 tid 100884 td 0xfffffe00a1198000 >> kdb_enter() at kdb_enter+0x3e/frame 0xffffff848e6bfd60 >> vpanic() at vpanic+0x147/frame 0xffffff848e6bfda0 >> kassert_panic() at kassert_panic+0x136/frame 0xffffff848e6bfe10 >> witness_checkorder() at witness_checkorder+0x289/frame 0xffffff848e6bfe90 >> __lockmgr_args() at __lockmgr_args+0x43e/frame 0xffffff848e6bffc0 >> vop_stdlock() at vop_stdlock+0x3c/frame 0xffffff848e6bffe0 >> VOP_LOCK1_APV() at VOP_LOCK1_APV+0xd0/frame 0xffffff848e6c0000 >> _vn_lock() at _vn_lock+0xab/frame 0xffffff848e6c0070 >> zfs_lookup() at zfs_lookup+0x392/frame 0xffffff848e6c0100 >> zfs_freebsd_lookup() at zfs_freebsd_lookup+0x6d/frame 0xffffff848e6c0240 >> VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0xc2/frame 0xffffff848e6c0260 >> vfs_cache_lookup() at vfs_cache_lookup+0xcf/frame 0xffffff848e6c02b0 >> VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xc2/frame 0xffffff848e6c02d0 >> lookup() at lookup+0x548/frame 0xffffff848e6c0350 >> nfsvno_namei() at nfsvno_namei+0x1a5/frame 0xffffff848e6c0400 >> nfsrvd_lookup() at nfsrvd_lookup+0x13a/frame 0xffffff848e6c06b0 >> nfsrvd_dorpc() at nfsrvd_dorpc+0xca5/frame 0xffffff848e6c08a0 >> nfssvc_program() at nfssvc_program+0x482/frame 0xffffff848e6c0a00 >> svc_run_internal() at svc_run_internal+0x1e9/frame 0xffffff848e6c0ba0 >> svc_thread_start() at svc_thread_start+0xb/frame 0xffffff848e6c0bb0 >> fork_exit() at fork_exit+0x84/frame 0xffffff848e6c0bf0 >> fork_trampoline() at fork_trampoline+0xe/frame 0xffffff848e6c0bf0 >> --- trap 0xc, rip = 0x800883b7a, rsp = 0x7fffffffd6c8, rbp = 0x7fffffffd970 --- >> > > > -- > Andriy Gapon -- wbr, pluknetReceived on Mon Feb 04 2013 - 11:13:34 UTC
This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:40:34 UTC