Re: at SVN r347375, terminating/restarting openvpn on tap causes panic

On 2019-05-09 14:22, Gleb Smirnoff wrote:
>   Michael,
> 
> On Thu, May 09, 2019 at 10:25:37AM -0500, Kyle Evans wrote:
> K> > #0  doadump () at src/sys/amd64/include/pcpu.h:241
> K> > #1  0xffffffff808393c8 in kern_reboot (howto=260) at
> K> > /usr/src/sys/kern/kern_shutdown.c:470
> K> > #2  0xffffffff80839826 in vpanic (fmt=<value optimized out>, ap=<value
> K> > optimized out>) at /usr/src/sys/kern/kern_shutdown.c:896
> K> > #3  0xffffffff80839663 in panic (fmt=<value optimized out>) at
> K> > /usr/src/sys/kern/kern_shutdown.c:823
> K> > #4  0xffffffff80c318a6 in trap_fatal (frame=0xfffffe0072b14510, eva=156)
> K> > at /usr/src/sys/amd64/amd64/trap.c:946
> K> > #5  0xffffffff80c31c59 in trap_pfault (frame=0xfffffe0072b14510,
> K> > usermode=0) at src/sys/amd64/include/cpufunc.h:423
> K> > #6  0xffffffff80c30fde in trap (frame=0xfffffe0072b14510) at
> K> > /usr/src/sys/amd64/amd64/trap.c:441
> K> > #7  0xffffffff80c0d4b5 in calltrap () at
> K> > /usr/src/sys/amd64/amd64/exception.S:232
> K> > #8  0xffffffff80a15377 in ip_output (m=<value optimized out>, opt=<value
> K> > optimized out>, ro=0x0, flags=0, imo=0xfffffe0072b14780, inp=0x0) at
> K> > /usr/src/sys/netinet/ip_output.c:362
> K> > #9  0xffffffff809ffea4 in igmp_intr (m=<value optimized out>) at
> K> > /usr/src/sys/netinet/igmp.c:3455
> K> > #10 0xffffffff80975a0f in netisr_dispatch_src (proto=2, source=<value
> K> > optimized out>, m=<value optimized out>) at /usr/src/sys/net/netisr.c:1122
> K> > #11 0xffffffff809fe07a in igmp_fasttimo () at
> K> > /usr/src/sys/netinet/igmp.c:496
> K> > #12 0xffffffff808c5854 in pffasttimo (arg=<value optimized out>) at
> K> > /usr/src/sys/kern/uipc_domain.c:521
> K> > #13 0xffffffff80853df3 in softclock_call_cc (c=0xffffffff813f7f48,
> K> > cc=0xffffffff814c9ac0, direct=0) at /usr/src/sys/kern/kern_timeout.c:731
> K> > #14 0xffffffff808542b9 in softclock (arg=0xffffffff814c9ac0) at
> K> > /usr/src/sys/kern/kern_timeout.c:869
> K> > #15 0xffffffff807fd0c4 in ithread_loop (arg=<value optimized out>) at
> K> > /usr/src/sys/kern/kern_intr.c:1129
> K> > #16 0xffffffff807f9f33 in fork_exit (callout=0xffffffff807fcef0
> K> > <ithread_loop>, arg=0xfffff800025f10a0, frame=0xfffffe0072b14ac0) at
> K> > /usr/src/sys/kern/kern_fork.c:1058
> K> > #17 0xffffffff80c0e4ae in fork_trampoline () at
> K> > /usr/src/sys/amd64/amd64/exception.S:995
> K> > #18 0x0000000000000000 in ?? ()
> K> >
> K> 
> K> Ah, I misread your backtrace (and forgot the proper tap detachment
> K> from my previous patch, so that's fixed/committed anyways). CC'ing
> K> Gleb for further triage as committer of r347375 that touched things in
> K> this path.
> 
> Michael, can you please dump a core and look at it in kgdb? Line 362 in
> ip_output() really belongs to part that had minimal change with r347375.
> So I need more details. Can you please print out in kgdb the following
> variables: imo, ifp, ia?
> 

This was a backtrace from kgdb. From frame 8, I see ..

(kgdb) frame 8
#8  0xffffffff80a15377 in ip_output (m=<value optimized out>, opt=<value
optimized out>, ro=0x0, flags=0, imo=0xfffffe0072b14780, inp=0x0) at
/usr/src/sys/netinet/ip_output.c:362
362                     IFP_TO_IA(ifp, ia, &in_ifa_tracker);
(kgdb) print imo
$1 = (struct ip_moptions *) 0xfffffe0072b14780
(kgdb) print ifp
$2 = (struct ifnet *) 0xfffff80004110000
(kgdb) print ia
$3 = <value optimized out>

(kgdb) print *imo
$4 = {imo_multicast_ifp = 0xfffff80004110000, imo_multicast_addr =
{s_addr = 1924220944}, imo_multicast_vif = 18446744073709551615,
imo_multicast_ttl = 1 '\001', imo_multicast_loop = 0 '\0',
  imo_num_memberships = 15350, imo_max_memberships = 63489,
imo_membership = 0xfffff80002c10d00, imo_mfilters = 0xfffff80002c10d00,
imo_epoch_ctx = {data = 0xfffffe0072b147b0}}
(kgdb) print *ifp
$5 = {if_link = {cstqe_next = 0x0}, if_clones = {le_next = 0x0, le_prev
= 0xfffff800040f9728}, if_groups = {cstqh_first = 0xfffff80002878d60,
cstqh_last = 0xfffff80002878d38},
  if_alloctype = 6 '\006', if_numa_domain = 255 '�', if_softc =
0xfffff80004197300, if_llsoftc = 0x0, if_l2com = 0xfffff800040fe800,
if_dname = 0xffffffff80e0bd14 "tap", if_dunit = 0,
  if_index = 4, if_index_reserved = 0, if_xname = 0xfffff80004110058
"tap0", if_description = 0x0, if_flags = 34818, if_drv_flags = 0,
if_capabilities = 524288, if_capenable = 524288,
  if_linkmib = 0x0, if_linkmiblen = 0, if_refcount = 1, if_type = 6
'\006', if_addrlen = 6 '\006', if_hdrlen = 14 '\016', if_link_state = 1
'\001', if_mtu = 1500, if_metric = 0,
  if_baudrate = 10000000, if_hwassist = 0, if_epoch = 10, if_lastchange
= {tv_sec = 1557408022, tv_usec = 929504}, if_snd = {ifq_head = 0x0,
ifq_tail = 0x0, ifq_len = 0, ifq_maxlen = 50,
    ifq_mtx = {lock_object = {lo_name = 0xfffff80004110058 "tap0",
lo_flags = 16973824, lo_data = 0, lo_witness = 0x0}, mtx_lock = 0},
ifq_drv_head = 0x0, ifq_drv_tail = 0x0,
    ifq_drv_len = 0, ifq_drv_maxlen = 0, altq_type = 0, altq_flags = 0,
altq_disc = 0x0, altq_ifp = 0xfffff80004110000, altq_enqueue = 0,
altq_dequeue = 0, altq_request = 0,
    altq_clfier = 0x0, altq_classify = 0, altq_tbr = 0x0, altq_cdnr =
0x0}, if_linktask = {ta_link = {stqe_next = 0x0}, ta_pending = 0,
ta_priority = 0,
    ta_func = 0xffffffff809494e0 <do_link_state_change>, ta_context =
0xfffff80004110000}, if_addr_lock = {lock_object = {lo_name =
0xffffffff80cc5adb "if_addr_lock", lo_flags = 16973824,
      lo_data = 0, lo_witness = 0x0}, mtx_lock = 0}, if_addrhead =
{cstqh_first = 0xfffff800040fee00, cstqh_last = 0xfffff800040fee28},
if_multiaddrs = {cstqh_first = 0xfffff8013bdc8780,
    cstqh_last = 0xfffff800020c7d80}, if_amcount = 0, if_addr =
0xfffff800040fee00, if_hw_addr = 0xfffff80002878d40, if_broadcastaddr =
0xffffffff80e0afc0 "������", if_afdata_lock = {
    lock_object = {lo_name = 0xffffffff80d13fcb "if_afdata", lo_flags =
16973824, lo_data = 0, lo_witness = 0x0}, mtx_lock = 0}, if_afdata =
0xfffff80004110208, if_afdata_initialized = 2,
  if_fib = 0, if_vnet = 0xfffff80002090040, if_home_vnet =
0xfffff80002090040, if_vlantrunk = 0x0, if_bpf = 0xfffff80004113280,
if_pcount = 0, if_bridge = 0x0, if_lagg = 0x0,
  if_pf_kif = 0x0, if_carp = 0x0, if_label = 0x0, if_netmap = 0x0,
if_output = 0xffffffff80959fe0 <ether_output>, if_input =
0xffffffff8095ace0 <ether_input>, if_bridge_input = 0,
  if_bridge_output = 0, if_bridge_linkstate = 0, if_start =
0xffffffff80961a70 <tunstart_l2>, if_ioctl = 0xffffffff80961750
<tunifioctl>, if_init = 0xffffffff80961a60 <tunifinit>,
  if_resolvemulti = 0xffffffff8095ad50 <ether_resolvemulti>, if_qflush =
0xffffffff8094d6e0 <if_qflush>, if_transmit = 0xffffffff80951ce0
<if_transmit>,
  if_reassign = 0xffffffff8095af40 <ether_reassign>, if_get_counter =
0xffffffff809497d0 <if_get_counter_default>, if_requestencap =
0xffffffff8095ae70 <ether_requestencap>,
  if_counters = 0xfffff80004110428, if_hw_tsomax = 65518,
if_hw_tsomaxsegcount = 35, if_hw_tsomaxsegsize = 2048, if_snd_tag_alloc
= 0, if_snd_tag_modify = 0, if_snd_tag_query = 0,
  if_snd_tag_free = 0, if_pcp = 255 '�', if_netdump_methods = 0x0,
if_epoch_ctx = {data = 0xfffff800041104c8}, if_ispare = 0xfffff800041104d8}

	imb