Re: kgdb crashing on a vmcore with dumptid = 0

On 5/4/18 10:49 PM, Eitan Adler wrote:
> gdb$ bt
> #0  thr_kill () at thr_kill.S:3
> #1  0x00000008035046b4 in __raise (s=0x6) at /usr/src/lib/libc/gen/raise.c:54
> #2  0x0000000803504629 in abort () at /usr/src/lib/libc/stdlib/abort.c:67
> #3  0x0000000000c805c6 in dump_core () at utils.c:284
> #4  0x0000000000c81920 in internal_vproblem (problem=0x24bd890
> <internal_error_problem>, file=0x15c560e "inferior.c", line=0x135,
> fmt=0x15780ac "%s: Assertion
>  `%s' failed.", ap=0x7fffffffb9e0) at utils.c:493
> #5  0x0000000000c806d8 in internal_verror (file=0x15c560e
> "inferior.c", line=0x135, fmt=0x15780ac "%s: Assertion `%s' failed.",
> ap=0x7fffffffb9e0) at utils.c:
> 518
> #6  0x00000000008363a8 in internal_error (file=0x15c560e "inferior.c",
> line=0x135, fmt=0x15780ac "%s: Assertion `%s' failed.") at
> common/errors.c:55
> #7  0x0000000000a8397e in find_inferior_pid (pid=0x0) at inferior.c:309
> #8  0x0000000000a83d54 in find_inferior_ptid (ptid=...) at inferior.c:323
> #9  0x0000000000c10049 in default_thread_architecture (ops=0x2702d90
> <dummy_target>, ptid=...) at target.c:3131
> #10 0x0000000000bfec2b in delegate_thread_architecture (self=0x2702d90
> <dummy_target>, arg1=...) at ./target-delegates.c:2525
> #11 0x0000000000bfec2b in delegate_thread_architecture (self=0x26eedd0
> <exec_ops>, arg1=...) at ./target-delegates.c:2525
> #12 0x0000000000bfec2b in delegate_thread_architecture (self=0x26ec280
> <kgdb_trgt_ops>, arg1=...) at ./target-delegates.c:2525
> #13 0x0000000000b53cba in get_thread_regcache (ptid=...) at regcache.c:439
> #14 0x0000000000b53d5e in get_current_regcache () at regcache.c:448
> #15 0x0000000000713077 in kgdb_trgt_open (arg=0x80410900e "vmcore.2",
> from_tty=0x1) at fbsd-kvm.c:335
> #16 0x0000000000bf3980 in open_target (args=0x80410900e "vmcore.2",
> from_tty=0x1, command=0x8041c8f40) at target.c:356
> #17 0x0000000000722c35 in do_sfunc (c=0x8041c8f40, args=0x80410900e
> "vmcore.2", from_tty=0x1) at cli/cli-decode.c:122
> #18 0x0000000000726b5a in cmd_func (cmd=0x8041c8f40, args=0x80410900e
> "vmcore.2", from_tty=0x1) at cli/cli-decode.c:1886
> #19 0x0000000000c450c7 in execute_command (p=0x804109015 "2",
> from_tty=0x1) at top.c:630
> #20 0x0000000000acd4a8 in catch_command_errors (command=0xc445f0
> <execute_command(char const*, int)>, arg=0x804109000 "target vmcore
> vmcore.2", from_tty=0x1)
> at main.c:378
> #21 0x0000000000accdf2 in captured_main_1 (context=0x7fffffffcfe8) at
> main.c:1124
> #22 0x0000000000aca62d in captured_main (data=0x7fffffffcfe8) at main.c:1146
> #23 0x0000000000aca508 in gdb_main (args=0x7fffffffcfe8) at main.c:1172
> #24 0x000000000050bd5b in main (argc=0x3, argv=0x7fffffffd558) at
> kgdb-main.c:410
> 
> gdb$ up 7
> #7  0x0000000000a8397e in find_inferior_pid (pid=0x0) at inferior.c:309
> 309       gdb_assert (pid != 0);
> gdb$ frame
> Stack level 7, frame at 0x7fffffffba50:
>  rip = 0xa8397e in find_inferior_pid (inferior.c:309); saved rip = 0xa83d54
>  called by frame at 0x7fffffffba60, caller of frame at 0x7fffffffba20
>  source language c++.
>  Arglist at 0x7fffffffba40, args: pid=0x0
>  Locals at 0x7fffffffba40, Previous frame's sp is 0x7fffffffba50
>  Saved registers:
>   rbp at 0x7fffffffba40, rip at 0x7fffffffba48
> pid = 0x0
> inf = 0x8415d5 <ptid_get_pid(ptid_t const&)+21>
> 
> 
> gdb$ list
> 304       struct inferior *inf;
> 305
> 306       /* Looking for inferior pid == 0 is always wrong, and indicative of
> 307          a bug somewhere else.  There may be more than one with pid == 0,
> 308          for instance.  */
> 309       gdb_assert (pid != 0);
> 310
> 311       for (inf = inferior_list; inf; inf = inf->next)
> 312         if (inf->pid == pid)
> 313           return inf;
> 
> 
> gdb$ p pid
> $1 = 0x0
> 
> 
> gdb$ p ptid
> $2 = {
>   m_pid = 0x0,
>   m_lwp = 0x0,
>   m_tid = 0x0
> }
> 
> gdb$ up
> #9  0x0000000000c10049 in default_thread_architecture (ops=0x2702d90
> <dummy_target>, ptid=...) at target.c:3131
> 3131      inferior *inf = find_inferior_ptid (ptid);
> gdb$ list
> 3126    }
> 3127
> 3128    static struct gdbarch *
> 3129    default_thread_architecture (struct target_ops *ops, ptid_t ptid)
> 3130    {
> 3131      inferior *inf = find_inferior_ptid (ptid);
> 3132      gdb_assert (inf != NULL);
> 3133      return inf->gdbarch;
> 3134    }
> 3135
> gdb$ p ops
> $4 = (target_ops *) 0x2702d90 <dummy_target>
> 
> gdb$ up
> #14 0x0000000000b53d5e in get_current_regcache () at regcache.c:448
> 448       return get_thread_regcache (inferior_ptid);
> gdb$ list
> 443     }
> 444
> 445     struct regcache *
> 446     get_current_regcache (void)
> 447     {
> 448       return get_thread_regcache (inferior_ptid);
> 449     }
> 450
> 451     /* See common/common-regcache.h.  */
> 452
> gdb$ p inferior_ptid
> $13 = {
>   m_pid = 0x0,
>   m_lwp = 0x0,
>   m_tid = 0x0
> }
> 
> 
> gdb$ up
> #15 0x0000000000713077 in kgdb_trgt_open (arg=0x80410900e "vmcore.2",
> from_tty=0x1) at fbsd-kvm.c:335
> 335             target_fetch_registers (get_current_regcache (), -1);
> gdb$ list
> 330                     kt = kgdb_thr_next(kt);
> 331             }
> 332             if (curkthr != 0)
> 333                     inferior_ptid = fbsd_vmcore_ptid(curkthr->tid);
> 334
> 335             target_fetch_registers (get_current_regcache (), -1);
> 336
> 337             reinit_frame_cache ();
> 338             print_stack_frame (get_selected_frame (NULL), 0,
> SRC_AND_LOC, 1);
> 339     }
> gdb$ p inferior_ptid
> $17 = {
>   m_pid = 0x0,
>   m_lwp = 0x0,
>   m_tid = 0x0
> }
> gdb$ p curkthr
> $18 = (kthr *) 0x0
> 
> gdb$ frame
> Stack level 15, frame at 0x7fffffffbd90:
>  rip = 0x713077 in kgdb_trgt_open (fbsd-kvm.c:335); saved rip = 0xbf3980
>  called by frame at 0x7fffffffbdc0, caller of frame at 0x7fffffffbc40
>  source language c++.
>  Arglist at 0x7fffffffbd80, args: arg=0x80410900e "vmcore.2", from_tty=0x1
>  Locals at 0x7fffffffbd80, Previous frame's sp is 0x7fffffffbd90
>  Saved registers:
>   rbp at 0x7fffffffbd80, rip at 0x7fffffffbd88
> arg = 0x80410900e "vmcore.2"
> from_tty = 0x1
> ops = 0x8043ef840
> inf = 0x80442de80
> old_chain = 0x804431820
> ti = 0x7fffffffd550
> kt = 0x0
> nkvm = 0x804363800
> temp = 0x8047f33b0 "/home/eax/crashes/aes_gpault_crash/vmcore.2"
> kernel = 0x8043dec80 "/home/eax/crashes/aes_gpault_crash/kernel/kernel"
> filename = 0x8047f33b0 "/home/eax/crashes/aes_gpault_crash/vmcore.2"
> ontop = 0x0
> 
> gdb$ p curkthr
> $19 = (kthr *) 0x0
> 
> which is coming from
> 
>         curkthr = kgdb_thr_lookup_tid(dumptid);
>         if (curkthr == NULL)
>                 curkthr = first;

Notice this fall back though.  If dumptid isn't valid this falls back to
just picking the first thread.  Instead, it seems that in your dump kgdb
didn't find any kthreads at all.  Is 'kernel' a stripped kernel without
any debug symbols?  You should try single-stepping kgdb_thr_init to see
if it finds valid offsets of structure members in 'struct thread' and
'struct proc'.

-- 
John Baldwin