Re: Deadlocks / hangs in ZFS

On Thu, Jul 12, 2018 at 02:42:29PM +0200, Alexander Leidinger wrote:

> __curthread () at ./machine/pcpu.h:230
> 230             __asm("movq %%gs:%1,%0" : "=r" (td)
> (kgdb) bt
> #0  __curthread () at ./machine/pcpu.h:230
> #1  doadump (textdump=1) at /usr/src/sys/kern/kern_shutdown.c:366
> #2  0xffffffff80485e11 in kern_reboot (howto=260) at  
> /usr/src/sys/kern/kern_shutdown.c:446
> #3  0xffffffff804863f3 in vpanic (fmt=<optimized out>, ap=0xfffffe0000457870)
>      at /usr/src/sys/kern/kern_shutdown.c:863
> #4  0xffffffff80486443 in panic (fmt=<unavailable>) at  
> /usr/src/sys/kern/kern_shutdown.c:790
> #5  0xffffffff8075279f in trap_fatal (frame=0xfffffe0000457a50,  
> eva=32) at /usr/src/sys/amd64/amd64/trap.c:892
> #6  0xffffffff80752812 in trap_pfault (frame=0xfffffe0000457a50,  
> usermode=<optimized out>)
>      at /usr/src/sys/amd64/amd64/trap.c:728
> #7  0xffffffff80751e1a in trap (frame=0xfffffe0000457a50) at  
> /usr/src/sys/amd64/amd64/trap.c:427
> #8  <signal handler called>
> #9  0xffffffff81391fbe in arc_check_uma_cache (lowest=-1011712)
>      at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:4532
> #10 arc_reclaim_thread (unused=<optimized out>)
>      at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:4657
> #11 0xffffffff8044ca74 in fork_exit (callout=0xffffffff81391b90  
> <arc_reclaim_thread>, arg=0x0,
>      frame=0xfffffe0000457c00) at /usr/src/sys/kern/kern_fork.c:1057
> #12 <signal handler called>
> (kgdb) up 9
> #9  0xffffffff81391fbe in arc_check_uma_cache (lowest=-1011712)
>      at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c:4532
> 4532                    lowest +=  
> uma_zone_get_free_size(zio_data_buf_cache[n]->kc_zone);
> (kgdb) list
> 4527            int                     iter = 4;
> 4528            int                     step = 1 << (SPA_MAXBLOCKSHIFT  
> - SPA_MINBLOCKSHIFT - 3);
> 4529            int                     n = (SPA_MAXBLOCKSIZE >>  
> SPA_MINBLOCKSHIFT) - 1;
> 4530
> 4531            while (n >= 0) {
> 4532                    lowest +=  
> uma_zone_get_free_size(zio_data_buf_cache[n]->kc_zone);
> 4533                    if (lowest >= 0)
> 4534                            return lowest;
> 4535                    n -= step;
> 4536                    if(--iter == 0) {
> (kgdb) print n
> $1 = 32767
> (kgdb) print zio_data_buf_cache[n]
> $2 = (kmem_cache_t *) 0x0
> (kgdb)

Very strange, zio_data_buf_cache[] can't be NULL, as asserted in
zio_init.