[This explores process crashes that happen during system shutdown, in a context not having MALLOC_PRODUCTION= . So assert failures are reported as the stopping points.] It looks like shutdown -p now, shutdown -r now, and the like can lead some processes to assert during their exit attempt, including a sshd failure (that I've not seen before), rpcbind, and nfsd. I show information about the observed asserts for those below. sshd hit an assert, failing slab == extent_slab_get(extent) : (gdb) bt #0 thr_kill () at thr_kill.S:4 #1 0x50927170 in __raise (s=6) at /usr/src/lib/libc/gen/raise.c:52 #2 0x50886cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67 #3 0x508834b0 in arena_dalloc (tsdn=<optimized out>, ptr=<optimized out>, tcache=<optimized out>, alloc_ctx=<optimized out>, slow_path=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h:315 #4 idalloctm (tsdn=0x500dd040, ptr=0x5008a180, tcache=0x500dd160, alloc_ctx=<optimized out>, is_internal=<optimized out>, slow_path=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h:118 #5 0x5087b0a4 in ifree (tsd=0x500dd040, ptr=0x5008a180, tcache=0x500dd160, slow_path=<optimized out>) at jemalloc_jemalloc.c:2590 #6 0x5087acac in __je_free_default (ptr=0x5008a180) at jemalloc_jemalloc.c:2784 #7 0x5087b294 in __free (ptr=0x5008a180) at jemalloc_jemalloc.c:2852 #8 0x10029464 in server_accept_loop (config_s=<optimized out>, sock_in=<optimized out>, sock_out=<optimized out>, newsock=<optimized out>) at /usr/src/crypto/openssh/sshd.c:1185 #9 main (ac=<optimized out>, av=0xffffde3c) at /usr/src/crypto/openssh/sshd.c:2009 . . . (gdb) up #2 0x50886cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67 67 (void)raise(SIGABRT); (gdb) up #3 0x508834b0 in arena_dalloc (tsdn=<optimized out>, ptr=<optimized out>, tcache=<optimized out>, alloc_ctx=<optimized out>, slow_path=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h:315 315 assert(slab == extent_slab_get(extent)); (gdb) list 310 rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); 311 extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, 312 rtree_ctx, (uintptr_t)ptr, true); 313 assert(szind == extent_szind_get(extent)); 314 assert(szind < SC_NSIZES); 315 assert(slab == extent_slab_get(extent)); 316 } 317 318 if (likely(slab)) { 319 /* Small allocation. */ More fully: 285 JEMALLOC_ALWAYS_INLINE void 286 arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, 287 alloc_ctx_t *alloc_ctx, bool slow_path) { 288 assert(!tsdn_null(tsdn) || tcache == NULL); 289 assert(ptr != NULL); 290 291 if (unlikely(tcache == NULL)) { 292 arena_dalloc_no_tcache(tsdn, ptr); 293 return; 294 } 295 296 szind_t szind; 297 bool slab; 298 rtree_ctx_t *rtree_ctx; 299 if (alloc_ctx != NULL) { 300 szind = alloc_ctx->szind; 301 slab = alloc_ctx->slab; 302 assert(szind != SC_NSIZES); 303 } else { 304 rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); 305 rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, 306 (uintptr_t)ptr, true, &szind, &slab); 307 } 308 309 if (config_debug) { 310 rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); 311 extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, 312 rtree_ctx, (uintptr_t)ptr, true); 313 assert(szind == extent_szind_get(extent)); 314 assert(szind < SC_NSIZES); 315 assert(slab == extent_slab_get(extent)); 316 } 317 318 if (likely(slab)) { 319 /* Small allocation. */ 320 tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, 321 slow_path); 322 } else { 323 arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path); 324 } 325 } rpcbind hit an assert, failing ret == sz_size2index_compute(size) : (gdb) bt #0 thr_kill () at thr_kill.S:4 #1 0x502f2170 in __raise (s=6) at /usr/src/lib/libc/gen/raise.c:52 #2 0x50251d04 in abort () at /usr/src/lib/libc/stdlib/abort.c:79 #3 0x5024f260 in sz_size2index_lookup (size=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:159 #4 sz_size2index (size=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:166 #5 imalloc_body (sopts=0xffffb360, dopts=0xffffb340, tsd=0x5009a018) at jemalloc_jemalloc.c:2066 #6 0x50244874 in imalloc (sopts=0xffffb360, dopts=0xffffb340) at /usr/src/contrib/jemalloc/include/jemalloc/internal/tsd.h:331 #7 0x50244fe8 in __calloc (num=1, size=96) at jemalloc_jemalloc.c:2498 #8 0x50265690 in svc_xprt_alloc () at /usr/src/lib/libc/rpc/svc.c:541 #9 0x502635f4 in makefd_xprt (fd=14, sendsize=9000, recvsize=9000) at /usr/src/lib/libc/rpc/svc_vc.c:250 #10 0x502644b4 in rendezvous_request (xprt=0x5004c000, msg=<optimized out>) at /usr/src/lib/libc/rpc/svc_vc.c:315 #11 0x50265a98 in svc_getreq_common (fd=<optimized out>) at /usr/src/lib/libc/rpc/svc.c:640 #12 0x50265d1c in svc_getreq_poll (pfdp=<optimized out>, pollretval=1) at /usr/src/lib/libc/rpc/svc.c:739 #13 0x10018568 in my_svc_run () at /usr/src/usr.sbin/rpcbind/rpcb_svc_com.c:1167 #14 0x10014ad8 in main (argc=<optimized out>, argv=<optimized out>) at /usr/src/usr.sbin/rpcbind/rpcbind.c:250 (gdb) up 3 #3 0x5024f260 in sz_size2index_lookup (size=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:159 159 assert(ret == sz_size2index_compute(size)); (gdb) print ret $1 = 0 154 JEMALLOC_ALWAYS_INLINE szind_t 155 sz_size2index_lookup(size_t size) { 156 assert(size <= SC_LOOKUP_MAXCLASS); 157 szind_t ret = (sz_size2index_tab[(size + (ZU(1) << SC_LG_TINY_MIN) - 1) 158 >> SC_LG_TINY_MIN]); 159 assert(ret == sz_size2index_compute(size)); 160 return ret; 161 } nfsd hit an assert, failing ret == sz_size2index_compute(size) (also, but a different caller of sz_size2index): (gdb) bt #0 thr_kill () at thr_kill.S:4 #1 0x502b2170 in __raise (s=6) at /usr/src/lib/libc/gen/raise.c:52 #2 0x50211cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67 #3 0x50206104 in sz_index2size_lookup (index=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:200 #4 sz_index2size (index=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:207 #5 ifree (tsd=0x50094018, ptr=0x50041028, tcache=0x50094138, slow_path=<optimized out>) at jemalloc_jemalloc.c:2583 #6 0x50205cac in __je_free_default (ptr=0x50041028) at jemalloc_jemalloc.c:2784 #7 0x50206294 in __free (ptr=0x50041028) at jemalloc_jemalloc.c:2852 #8 0x50287ec8 in ns_src_free (src=0x50329004, srclistsize=<optimized out>) at /usr/src/lib/libc/net/nsdispatch.c:452 #9 ns_dbt_free (dbt=0x50329000) at /usr/src/lib/libc/net/nsdispatch.c:436 #10 vector_free (vec=0x50329000, count=<optimized out>, esize=12, free_elem=<optimized out>) at /usr/src/lib/libc/net/nsdispatch.c:253 #11 nss_atexit () at /usr/src/lib/libc/net/nsdispatch.c:578 #12 0x5028d958 in __cxa_finalize (dso=0x0) at /usr/src/lib/libc/stdlib/atexit.c:240 #13 0x502117f8 in exit (status=0) at /usr/src/lib/libc/stdlib/exit.c:74 #14 0x10013f9c in child_cleanup (signo=<optimized out>) at /usr/src/usr.sbin/nfsd/nfsd.c:969 #15 <signal handler called> #16 0x00000000 in ?? () (gdb) up 3 #3 0x50206104 in sz_index2size_lookup (index=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:200 200 assert(ret == sz_index2size_compute(index)); (ret is optimized out.) 197 JEMALLOC_ALWAYS_INLINE size_t 198 sz_index2size_lookup(szind_t index) { 199 size_t ret = (size_t)sz_index2size_tab[index]; 200 assert(ret == sz_index2size_compute(index)); 201 return ret; 202 } Booting and immediately trying something like: service nfsd stop did not lead to a failure. But may be after a while it would and be less drastic than a reboot or power down. === Mark Millard marklmi at yahoo.com ( dsl-only.net went away in early 2018-Mar)Received on Wed May 06 2020 - 21:58:06 UTC
This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:41:23 UTC