panic: not waking up swapper

From: Robert Watson <rwatson_at_FreeBSD.org>
Date: Fri, 22 Aug 2008 12:07:28 +0100 (BST)
I ran into the above panic on a VMware image running -CURRENT from yesterday 
(with some local but likely unrelated changes) during a make -j4 buildworld. 
I was doing things in other VMs on the same host that were I/O-intensive, so 
the possibility of delayed I/O and other weird timing effects is non-trivial. 
Some details below; I do have a crashdump.

Robert N M Watson
Computer Laboratory
University of Cambridge

cinnamon-capabilities# ddb capture -M /var/crash/vmcore.2 -N 
/boot/kernel/kernel.symbols print
db> bt

Tracing pid 6922 tid 100101 td 0xc2cc1230
kdb_enter(c0b54ec6,c0b54ec6,c0b56138,cd112bac,0,...) at kdb_enter+0x3a
panic(c0b56138,4,c0b560b7,31b) at panic+0x12c
thread_unsuspend_one(c30ec690,0,c0b560b7,332,ffffffff,...) at 
thread_unsuspend_one+0xa3
thread_unsuspend(c30e9d0c,0,c0b54f35,84e,0,...) at thread_unsuspend+0x96
tdsignal(c30e9d0c,0,13,0,cd112c5c,...) at tdsignal+0x5cd
psignal(c30e9d0c,13,13,66c,13,...) at psignal+0x28
killpg1(0,c0b8b894,349,28170340,2821f9e0,...) at killpg1+0x294
kill(c2cc1230,cd112cf8,8,cd112d38,c0c2d76c,...) at kill+0x145
syscall(cd112d38) at syscall+0x2b3
Xint0x80_syscall() at Xint0x80_syscall+0x20
--- syscall (37, FreeBSD ELF32, kill), eip = 0x2820a757, esp = 0xbfbfe6bc, ebp 
= 0xbfbfe6d8 ---

(kgdb) bt
#0  doadump () at pcpu.h:221
#1  0xc0496779 in db_fncall (dummy1=-1058895488, dummy2=0, dummy3=3,
     dummy4=0xcd112984 "")
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/ddb/db_command.c:516
#2  0xc0496d28 in db_command (last_cmdp=0xc0c699f0, cmd_table=0x0, dopager=1)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/ddb/db_command.c:413
#3  0xc0496e5a in db_command_loop ()
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/ddb/db_command.c:466
#4  0xc049864d in db_trap (type=3, code=0)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/ddb/db_main.c:228
#5  0xc07dee56 in kdb_trap (type=3, code=0, tf=0xcd112b2c)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/subr_kdb.c:534
#6  0xc0abd69b in trap (frame=0xcd112b2c)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/i386/i386/trap.c:696
#7  0xc0aa147b in calltrap ()
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/i386/i386/exception.s:165
---Type <return> to continue, or q <return> to quit---
#8  0xc07defda in kdb_enter (why=0xc0b54ec6 "panic", msg=0xc0b54ec6 "panic")
     at cpufunc.h:70
#9  0xc07b1b7c in panic (fmt=0xc0b56138 "not waking up swapper")
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/kern_shutdown.c:556
#10 0xc07bed63 in thread_unsuspend_one (td=0xc30ec690)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/kern_thread.c:801
#11 0xc07bee06 in thread_unsuspend (p=0xc30e9d0c)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/kern_thread.c:820
#12 0xc07b5aad in tdsignal (p=0xc30e9d0c, td=0xc30ec690, sig=19, ksi=0x0)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/kern_sig.c:2129
#13 0xc07b62d8 in psignal (p=0xc30e9d0c, sig=19)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/kern_sig.c:1925
#14 0xc07b6a04 in killpg1 (td=0xc2cc1230, sig=19, pgid=Variable "pgid" is not 
available.
)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/kern_sig.c:1653
#15 0xc07b6bb5 in kill (td=0xc2cc1230, uap=0xcd112cf8)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/kern_sig.c:1701
---Type <return> to continue, or q <return> to quit---
#16 0xc0abcdc3 in syscall (frame=0xcd112d38)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/i386/i386/trap.c:1104
#17 0xc0aa14e0 in Xint0x80_syscall ()
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/i386/i386/exception.s:261
#18 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb)
(kgdb) frame 10
#10 0xc07bed63 in thread_unsuspend_one (td=0xc30ec690)
     at 
/usr/home/robert/p4/projects/trustedbsd/capabilities/src/sys/kern/kern_thread.c:801
801			panic("not waking up swapper");
(kgdb) print *td
$1 = {td_lock = 0xc0c9f480, td_proc = 0xc30e9d0c, td_plist = {tqe_next = 0x0,
     tqe_prev = 0xc30e9d14}, td_runq = {tqe_next = 0x0, tqe_prev = 0xc0c9f5b8},
   td_slpq = {tqe_next = 0x0, tqe_prev = 0xc24a2870}, td_lockq = {
     tqe_next = 0x0, tqe_prev = 0xcd0e0a30}, td_cpuset = 0xc24a3e38,
   td_sel = 0xc2e07000, td_sleepqueue = 0xc24a2870, td_turnstile = 0xc26acd70,
   td_umtxq = 0xc2ccde00, td_tid = 100117, td_sigqueue = {sq_signals = {
       __bits = {262144, 0, 0, 0}}, sq_kill = {__bits = {262144, 0, 0, 0}},
     sq_list = {tqh_first = 0x0, tqh_last = 0xc30ec6f0}, sq_proc = 0xc30e9d0c,
     sq_flags = 1}, td_flags = 4327488, td_inhibitors = 4, td_pflags = 0,
   td_dupfd = 0, td_sqqueue = 0, td_wchan = 0x0, td_wmesg = 0x0,
   td_lastcpu = 0 '\0', td_oncpu = 255 '?', td_owepreempt = 0 '\0',
   td_tsqueue = 0 '\0', td_locks = 0, td_rw_rlocks = 0, td_lk_slocks = 0,
   td_blocked = 0x0, td_lockname = 0x0, td_contested = {lh_first = 0x0},
   td_sleeplocks = 0xc0dd3c78, td_intr_nesting_level = 0, td_pinned = 0,
   td_ucred = 0xc2dc4000, td_estcpu = 0, td_slptick = 5415917, td_ru = {
     ru_utime = {tv_sec = 0, tv_usec = 0}, ru_stime = {tv_sec = 0,
       tv_usec = 0}, ru_maxrss = 1908, ru_ixrss = 1184, ru_idrss = 4400,
     ru_isrss = 512, ru_minflt = 265, ru_majflt = 0, ru_nswap = 1,
     ru_inblock = 0, ru_oublock = 0, ru_msgsnd = 0, ru_msgrcv = 0,
     ru_nsignals = 0, ru_nvcsw = 65, ru_nivcsw = 19}, td_incruntime = 0,
   td_runtime = 53752069, td_pticks = 0, td_sticks = 0, td_iticks = 0,
   td_uticks = 0, td_uuticks = 0, td_usticks = 0, td_intrval = 0,
   td_oldsigmask = {__bits = {0, 0, 0, 0}}, td_sigmask = {__bits = {0, 0, 0,
---Type <return> to continue, or q <return> to quit---
       0}}, td_generation = 84, td_sigstk = {ss_sp = 0x0, ss_size = 0,
     ss_flags = 4}, td_xsig = 0, td_profil_addr = 0, td_profil_ticks = 0,
   td_name = "vi\000\000\000l", '\0' <repeats 13 times>, td_fpop = 0x0,
   td_rqindex = 32 ' ', td_base_pri = 128 '\200', td_priority = 128 '\200',
   td_pri_class = 3 '\003', td_user_pri = 128 '\200',
   td_base_user_pri = 128 '\200', td_pcb = 0xcd157d90,
   td_state = TDS_INHIBITED, td_retval = {0, 0}, td_slpcallout = {c_links = {
       sle = {sle_next = 0x0}, tqe = {tqe_next = 0x0, tqe_prev = 0xc2097be0}},
     c_time = 5305740, c_arg = 0xc30ec690,
     c_func = 0xc07e6e10 <sleepq_timeout>, c_lock = 0x0, c_flags = 16,
     c_cpu = 0}, td_frame = 0xcd157d38, td_kstack_obj = 0xc30d1174,
   td_kstack = 3440730112, td_kstack_pages = 2, td_altkstack_obj = 0x0,
   td_altkstack = 0, td_altkstack_pages = 0, td_critnest = 1, td_md = {
     md_spinlock_count = 1, md_saved_flags = 582}, td_sched = 0xc30ec890,
   td_ar = 0x0, td_syscalls = 29337, td_lprof = {{lh_first = 0x0}, {
       lh_first = 0x0}}, td_dtrace = 0x0, td_errno = 0}
kgdb) print td->td_proc->p_pid
$2 = 7584
(kgdb) print *td->td_proc
$3 = {p_list = {le_next = 0xc2db9a70, le_prev = 0xc30e97d4}, p_threads = {
     tqh_first = 0xc30ec690, tqh_last = 0xc30ec698}, p_slock = {lock_object = {
       lo_name = 0xc0b53b29 "process slock", lo_flags = 720896, lo_data = 0,
       lo_witness = 0x0}, mtx_lock = 3268153904}, p_ucred = 0xc2dc4000,
   p_fd = 0xc2e3c700, p_fdtol = 0x0, p_stats = 0xc2cba700,
   p_limit = 0xc2dc4100, p_limco = {c_links = {sle = {sle_next = 0x0}, tqe = {
         tqe_next = 0x0, tqe_prev = 0x0}}, c_time = 0, c_arg = 0x0, c_func = 0,
     c_lock = 0xc30e9d94, c_flags = 0, c_cpu = 0}, p_sigacts = 0xc2fcb000,
   p_flag = 86018, p_state = PRS_NORMAL, p_pid = 7584, p_hash = {le_next = 0x0,
     le_prev = 0xc249c280}, p_pglist = {le_next = 0x0, le_prev = 0xc2ccd608},
   p_pptr = 0xc2db9a70, p_sibling = {le_next = 0x0, le_prev = 0xc2db9af4},
   p_children = {lh_first = 0x0}, p_mtx = {lock_object = {
       lo_name = 0xc0b53b1c "process lock", lo_flags = 21168128, lo_data = 0,
       lo_witness = 0xc2471340}, mtx_lock = 3268153904}, p_ksi = 0xc2674690,
   p_sigqueue = {sq_signals = {__bits = {0, 0, 0, 0}}, sq_kill = {__bits = {0,
         0, 0, 0}}, sq_list = {tqh_first = 0x0, tqh_last = 0xc30e9dcc},
     sq_proc = 0xc30e9d0c, sq_flags = 1}, p_oppid = 0, p_vmspace = 0xc2975b60,
   p_swtick = 5638504, p_realtimer = {it_interval = {tv_sec = 0, tv_usec = 0},
     it_value = {tv_sec = 0, tv_usec = 0}}, p_ru = {ru_utime = {tv_sec = 0,
       tv_usec = 0}, ru_stime = {tv_sec = 0, tv_usec = 0}, ru_maxrss = 0,
     ru_ixrss = 0, ru_idrss = 0, ru_isrss = 0, ru_minflt = 0, ru_majflt = 0,
     ru_nswap = 0, ru_inblock = 0, ru_oublock = 0, ru_msgsnd = 0,
     ru_msgrcv = 0, ru_nsignals = 0, ru_nvcsw = 0, ru_nivcsw = 0}, p_rux = {
---Type <return> to continue, or q <return> to quit---
     rux_runtime = 53752069, rux_uticks = 1, rux_sticks = 3, rux_iticks = 0,
     rux_uu = 0, rux_su = 0, rux_tu = 0}, p_crux = {rux_runtime = 0,
     rux_uticks = 0, rux_sticks = 0, rux_iticks = 0, rux_uu = 0, rux_su = 0,
     rux_tu = 0}, p_profthreads = 0, p_exitthreads = 0, p_traceflag = 0,
   p_tracevp = 0x0, p_tracecred = 0x0, p_textvp = 0xc29fc96c, p_lock = 0 '\0',
   p_sigiolst = {slh_first = 0x0}, p_sigparent = 20, p_sig = 0, p_code = 0,
   p_stops = 0, p_stype = 0, p_step = 0 '\0', p_pfsflags = 0 '\0',
   p_nlminfo = 0x0, p_aioinfo = 0x0, p_singlethread = 0x0, p_suspcount = 0,
   p_xthread = 0x0, p_boundary_count = 0, p_pendingcnt = 0, p_itimers = 0x0,
   p_magic = 3203398350, p_osrel = 800045,
   p_comm = "vi\000\000\000l", '\0' <repeats 13 times>, p_pgrp = 0xc2ccd600,
   p_sysent = 0xc0c5fb40, p_args = 0xc2dea140,
   p_cpulimit = 9223372036854775807, p_nice = 0 '\0', p_fibnum = 0,
   p_xstat = 19, p_klist = {kl_list = {slh_first = 0x0},
     kl_lock = 0xc0789660 <knlist_mtx_lock>,
     kl_unlock = 0xc0789690 <knlist_mtx_unlock>,
     kl_locked = 0xc078aa10 <knlist_mtx_locked>, kl_lockarg = 0xc30e9d94},
   p_numthreads = 1, p_md = {md_ldt = 0x0}, p_itcallout = {c_links = {sle = {
         sle_next = 0x0}, tqe = {tqe_next = 0x0, tqe_prev = 0x0}}, c_time = 0,
     c_arg = 0x0, c_func = 0, c_lock = 0x0, c_flags = 16, c_cpu = 0},
   p_acflag = 0, p_peers = 0x0, p_leader = 0xc30e9d0c, p_emuldata = 0x0,
   p_label = 0x0, p_sched = 0xc30e9fa8, p_ktr = {stqh_first = 0x0,
     stqh_last = 0xc30e9f98}, p_mqnotifier = {lh_first = 0x0}, p_dtrace = 0x0}
Received on Fri Aug 22 2008 - 09:07:29 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:39:34 UTC