RE: Processes blocked on getblk or ufs

From: Robin P. Blanchard <robin.blanchard_at_gactr.uga.edu>
Date: Fri, 6 Feb 2004 11:26:40 -0500
Ok. I just induced a getblk with installworld (again stuck in makewhatis). I
logged the ddb session below, during which I manually induced a dump. The
dump allegedly completely sucessfully; but the box never rebooted. I had to
manually (hard) reboot it. Once it came back up, savecore recorded the dump
as expected. Unfortunately, it is apparently useless (at least to me). I have
saved kernel.debug as well as vmcore for anyone who thinks they can use this.
I'm now at a loss as to how to proceed.
Thanks in advance.

# gdb -k ./kernel.debug ./vmcore.1 
GNU gdb 5.2.1 (FreeBSD)
Copyright 2002 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-undermydesk-freebsd"...
/usr/local/var/tmp/getblk/./vmcore.1: Unknown error: 0.

# sysctl debug.enter_debugger=ddb
debug.enter_debuDgger: ebugger("debug.enter_debugger")
Stopped at      Debugger+0x55:  xchgl   %ebx,in_Debugger.0
db> ps
  pid   proc     uarea   uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
 4003 c65a0dc0 ec21a000    0  3984  4003 0004002 [CPU 0] sysctl
 3984 c61f7a50 e186a000    0   606  3984 0004002 [SLP]wait 0xc61f7a50] bash
 3982 c65a0000 ec212000    0   612  3982 0004002 [SLP]ttyin 0xc6479840] bash
 3953 c6641370 ec27e000    0  3952 87328 0004002 [SLP]getblk 0xd2ac6034]
makewhatis
 3952 c704f1b8 ec2f4000    0  3951 87328 0004002 [SLP]wait 0xc704f1b8] make
 3951 c66411b8 ec27d000    0  3946 87328 0004002 [SLP]wait 0xc66411b8] sh
 3946 c642e1b8 e9f53000    0 87461 87328 0004002 [SLP]wait 0xc642e1b8] make
87461 c659e898 ec1ea000    0 87454 87328 0004002 [SLP]wait 0xc659e898] sh
87454 c704fdc0 ec31f000    0 87453 87328 0004002 [SLP]wait 0xc704fdc0] make
87453 c659e528 ec1e8000    0 87403 87328 0004002 [SLP]wait 0xc659e528] sh
87403 c65a0370 ec214000    0 87402 87328 0004002 [SLP]wait 0xc65a0370] make
87402 c64896e0 ec1c1000    0 87335 87328 0004002 [SLP]wait 0xc64896e0] sh
87335 c7052c08 ec327000    0 87333 87328 0004002 [SLP]wait 0xc7052c08] make
87333 c659ec08 ec1ec000    0 87328 87328 0004002 [SLP]wait 0xc659ec08] sh
87328 c704f6e0 ec31b000    0   614 87328 0004002 [SLP]wait 0xc704f6e0] make
  614 c6641898 ec281000    0   612   614 0004002 [SLP]wait 0xc6641898] bash
  612 c66416e0 ec280000    0   514   612 0000100 [CV]select 0xc0737ce4] sshd
  606 c663ec08 ec27a000    0     1   606 0004102 [SLP]wait 0xc663ec08] login
  605 c663e898 ec230000    0     1   605 0004002 [SLP]ttyin 0xc6268610] getty
  604 c65a0c08 ec219000    0     1   604 0004002 [SLP]ttyin 0xc6411810] getty
  603 c663e6e0 ec22f000    0     1   603 0004002 [SLP]ttyin 0xc6268e10] getty
  602 c663e528 ec22e000    0     1   602 0004002 [SLP]ttyin 0xc6479410] getty
  601 c663e370 ec22d000    0     1   601 0004002 [SLP]ttyin 0xc6147e10] getty
  542 c642e370 e9f54000    0     1   542 0000000 [SLP]nanslp 0xc070a01c] cron
  527 c659ea50 ec1eb000   25     1   527 0000100 [SLP]pause 0xc64c2000]
sendmail
  521 c61f7dc0 e186c000    0     1   521 0000100 [CV]select 0xc0737ce4]
sendmail
  514 c64301b8 e9f80000    0     1   514 0000100 [CV]select 0xc0737ce4] sshd
  490 c6430dc0 e9f87000    0     1   490 0000000 [CV]select 0xc0737ce4] ntpd
  451 c659edc0 ec211000    0   443   443 0000100 [CV]select 0xc0737ce4]
rpc.lockd
  443 c65a0528 ec215000    0     1   443 0000000 [CV]select 0xc0737ce4]
rpc.lockd
  438 c6430898 e9f84000    0     1   438 0000000 [CV]select 0xc0737ce4]
rpc.statd
  359 c65a0898 ec217000    1     1   359 0000100 [CV]select 0xc0737ce4]
rpcbind
  336 c6489528 ec1c0000    0     1   336 0000000 [CV]select 0xc0737ce4]
syslogd
  228 c61f7898 e1869000    0     1   228 0000000 [SLP]pause 0xc6467000]
adjkerntz
   81 c642e6e0 e9f56000    0     0     0 0000204 [SLP]- 0xc07433ac] nfsiod 3
   80 c642e898 e9f57000    0     0     0 0000204 [SLP]- 0xc07433a8] nfsiod 2
   79 c642ea50 e9f58000    0     0     0 0000204 [SLP]- 0xc07433a4] nfsiod 1
   78 c642ec08 e9f59000    0     0     0 0000204 [SLP]- 0xc07433a0] nfsiod 0
   77 c642edc0 e9f5a000    0     0     0 0000204 [SLP]syncer 0xc07099e0]
syncer
   76 c6430000 e9f5b000    0     0     0 0000204 [SLP]vlruwt 0xc6430000]
vnlru
   75 c61a6a50 e1834000    0     0     0 0000204 [SLP]psleep 0xc073818c]
bufdaemon
   74 c61a6c08 e1835000    0     0     0 000020c [SLP]pgzero 0xc0749bc8]
pagezero
   73 c61a6dc0 e1836000    0     0     0 0000204 [SLP]psleep 0xc0749c20]
vmdaemon
   72 c61f5000 e1837000    0     0     0 0000204 [SLP]psleep 0xc0749c0c]
pagedaemon
    9 c61f51b8 e1838000    0     0     0 0000204 [SLP]aifthd 0xc61f51b8]
aac0aif 
   71 c61f5370 e1839000    0     0     0 0000204 [LOCK  Giant c22627c0] swi0:
tty:sio
    8 c61f5528 e183a000    0     0     0 0000204 [SLP]actask 0xc070196c]
acpi_task2
    7 c61f56e0 e183b000    0     0     0 0000204 [SLP]actask 0xc070196c]
acpi_task1
    6 c61f5898 e183c000    0     0     0 0000204 [SLP]actask 0xc070196c]
acpi_task0
   70 c61f5a50 e183d000    0     0     0 0000204 new [IWAIT] swi6:+
   69 c61f5c08 e183e000    0     0     0 0000204 [IWAIT] swi7: task queue
   68 c61f5dc0 e183f000    0     0     0 0000204 [IWAIT] swi3: cambio
   67 c61f7000 e1840000    0     0     0 0000204 new [IWAIT] swi2: camnet
   66 c61f71b8 e1841000    0     0     0 0000204 [IWAIT] swi7: acpitaskq
   65 c61f7370 e1842000    0     0     0 0000204 [IWAIT] swi5:+
    5 c61981b8 e17de000    0     0     0 0000204 [SLP]tqthr 0xc0711d48]
taskqueue
   64 c6198370 e17df000    0     0     0 0000204 [SLP]- 0xc0701bc0] random
    4 c6198528 e17e0000    0     0     0 0000204 [SLP]- 0xc07046c0] g_down
    3 c61986e0 e17e1000    0     0     0 0000204 [SLP]- 0xc07046bc] g_up
    2 c6198898 e17e2000    0     0     0 0000204 [SLP]- 0xc07046b4] g_event
   63 c6198a50 e17e3000    0     0     0 0000204 [IWAIT] swi1: net
   62 c6198c08 e17e4000    0     0     0 0000204 new [IWAIT] swi4: vm
   61 c6198dc0 e17e5000    0     0     0 000020c [LOCK  Giant c22627c0] swi8:
tty:sio clock
   60 c61a6000 e17e6000    0     0     0 0000204 new [IWAIT] irq0: clk
   59 c61a61b8 e17e7000    0     0     0 0000204 new [IWAIT] irq47:
   58 c61a6370 e17e8000    0     0     0 0000204 new [IWAIT] irq46:
   57 c61a6528 e17e9000    0     0     0 0000204 new [IWAIT] irq45:
   56 c61a66e0 e17ea000    0     0     0 0000204 new [IWAIT] irq44:
   55 c61a6898 e1833000    0     0     0 0000204 new [IWAIT] irq43:
   54 c6181a50 e17ad000    0     0     0 0000204 new [IWAIT] irq42:
   53 c6181c08 e17ae000    0     0     0 0000204 new [IWAIT] irq41:
   52 c6181dc0 e17af000    0     0     0 0000204 new [IWAIT] irq40:
   51 c6195000 e17b0000    0     0     0 0000204 new [IWAIT] irq39:
   50 c61951b8 e17b1000    0     0     0 0000204 new [IWAIT] irq38:
   49 c6195370 e17b2000    0     0     0 0000204 new [IWAIT] irq37:
   48 c6195528 e17b3000    0     0     0 0000204 new [IWAIT] irq36:
   47 c61956e0 e17b4000    0     0     0 0000204 new [IWAIT] irq35:
   46 c6195898 e17b5000    0     0     0 0000204 new [IWAIT] irq34:
   45 c6195a50 e17b6000    0     0     0 0000204 new [IWAIT] irq33:
   44 c6195c08 e17db000    0     0     0 0000204 new [IWAIT] irq32:
   43 c6195dc0 e17dc000    0     0     0 0000204 new [IWAIT] irq31:
   42 c6198000 e17dd000    0     0     0 0000204 new [IWAIT] irq30: aac0
   41 c6171528 e177d000    0     0     0 0000204 [LOCK  Giant c22627c0]
irq29: bge1
   40 c61716e0 e177e000    0     0     0 0000204 [LOCK  Giant c22627c0]
irq28: bge0
   39 c6171898 e177f000    0     0     0 0000204 new [IWAIT] irq27:
   38 c6171a50 e1780000    0     0     0 0000204 new [IWAIT] irq26:
   37 c6171c08 e1781000    0     0     0 0000204 new [IWAIT] irq25:
   36 c6171dc0 e1782000    0     0     0 0000204 new [IWAIT] irq24:
   35 c6181000 e1783000    0     0     0 0000204 new [IWAIT] irq23:
   34 c61811b8 e17a8000    0     0     0 0000204 new [IWAIT] irq22:
   33 c6181370 e17a9000    0     0     0 0000204 new [IWAIT] irq21:
   32 c6181528 e17aa000    0     0     0 0000204 new [IWAIT] irq20:
   31 c61816e0 e17ab000    0     0     0 0000204 new [IWAIT] irq19:
   30 c6181898 e17ac000    0     0     0 0000204 new [IWAIT] irq18:
   29 c22691b8 dfd45000    0     0     0 0000204 new [IWAIT] irq17:
   28 c2269370 dfd46000    0     0     0 0000204 new [IWAIT] irq16:
   27 c2269528 dfd47000    0     0     0 0000204 new [IWAIT] irq15: ata1
   26 c22696e0 dfd48000    0     0     0 0000204 [IWAIT] irq14: ata0
   25 c2269898 dfd6d000    0     0     0 0000204 new [IWAIT] irq13:
   24 c2269a50 dfd6e000    0     0     0 0000204 new [IWAIT] irq12: psm0
   23 c2269c08 dfd6f000    0     0     0 0000204 new [IWAIT] irq11:
   22 c2269dc0 dfd70000    0     0     0 0000204 new [IWAIT] irq10:
   21 c6171000 e177a000    0     0     0 0000204 new [IWAIT] irq9: acpi0
   20 c61711b8 e177b000    0     0     0 0000204 new [IWAIT] irq8: rtc
   19 c6171370 e177c000    0     0     0 0000204 new [IWAIT] irq7:
   18 c2260000 dfcf3000    0     0     0 0000204 [IWAIT] irq6: fdc0
   17 c22601b8 dfd3c000    0     0     0 0000204 new [IWAIT] irq5:
   16 c2260370 dfd3d000    0     0     0 0000204 new [IWAIT] irq4: sio0
   15 c2260528 dfd3e000    0     0     0 0000204 new [IWAIT] irq3: sio1
   14 c22606e0 dfd3f000    0     0     0 0000204 new [IWAIT] irq2:
   13 c2260898 dfd40000    0     0     0 0000204 [IWAIT] irq1: atkbd0
   12 c2260a50 dfd41000    0     0     0 000020c [Can run] idle: cpu0
   11 c2260c08 dfd42000    0     0     0 000020c [CPU 1] idle: cpu1
    1 c2260dc0 dfd43000    0     0     1 0004200 [SLP]wait 0xc2260dc0] init
   10 c2269000 dfd44000    0     0     0 0000204 [CV]ktrace 0xc0707c04]
ktrace
    0 c0704760 c0c1f000    0     0     0 0000200 [SLP]sched 0xc0704760]
swapper
db> tr 3953
sched_switch(c663f930,2,c06b53a5,1d2,cec0aa89) at sched_switch+0x165
mi_switch(1,50,c06b53a5,ca,1) at mi_switch+0x298
msleep(d2ac6034,c070864c,50,c06bb742,0) at msleep+0x500
acquire(ec248a38,2000020,600,e7,c06bb3c9) at acquire+0xae
debuglockmgr(d2ac6034,2090022,c684f6f0,c663f930,c06af5fb) at
debuglockmgr+0x477
getblk(c684f6f0,0,0,800,0) at getblk+0x1a7
breadn(c684f6f0,0,0,800,0) at breadn+0x52
bread(c684f6f0,0,0,800,0) at bread+0x4c
ffs_read(ec248be4,c06bd3bc,c663f930,c06bd39e,20e) at ffs_read+0x43f
vn_read(c6478220,ec248c80,c66d3380,0,c663f930) at vn_read+0x215
dofileread(c663f930,c6478220,4,80e4000,4000) at dofileread+0xdc
read(c663f930,ec248d14,c,43a,3) at read+0x6b
syscall(2f,2f,2f,806c138,4000) at syscall+0x2a0
Xint0x80_syscall() at Xint0x80_syscall+0x1d
--- syscall (3, FreeBSD ELF32, read), eip = 0x8061a4f, esp = 0xbfbfd3ac, ebp
= 0 xbfbfd3c8 ---
db> show lockedvnods
Locked vnodes
0xc684f6f0: tag ufs, type VREG, usecount 1, writecount 0, refcount 1, flags
(VV_OBJBUF), lock type ufs:
 EXCL (count 1) by thread 0xc663f930 (pid 3953)
        ino 1973, on dev aacd0s1f (4, 25)
db> panic
panic: from debugger
at line 450 in file /usr/src/sys/ddb/db_command.c
cpuid = 1;
Stack backtrace:
backtrace(c06956e5,1,1c2,c06a8336,100) at backtrace+0x17
__panic(c06a8336,1c2,c06a8328,e1866a4c,c048d872) at __panic+0x15d
db_panic(c0658bc5,0,ffffffff,e18669c0,e18669bc) at db_panic+0x22
db_command(c06fff40,c06d7900,c06d1494,c06d1498,3) at db_command+0x272
db_command_loop(c0658bc5,c61f8e70,0,e1866a98,0) at db_command_loop+0x95
db_trap(3,0,0,46,0) at db_trap+0xc5
kdb_trap(3,0,e1866b18,157,0) at kdb_trap+0x14c
trap(c06e0018,10,c06b0010,e1866c10,0) at trap+0x4b8
calltrap() at calltrap+0x5
--- trap 0x3, eip = 0xc0658bc5, esp = 0xe1866b58, ebp = 0xe1866b64 ---
Debugger(c06a880f,c06b7e5b,3,e1866c10,626464) at Debugger+0x55
sysctl_debug_enter_debugger(c06d7be0,0,0,e1866c10,e1866c10) at
sysctl_debug_enter_debugger+0xac
sysctl_root(0,e1866cac,2,e1866c10,c61f8e70) at sysctl_root+0x17b
userland_sysctl(c61f8e70,e1866cac,2,0,0) at userland_sysctl+0x14d
__sysctl(c61f8e70,e1866d14,18,43a,6) at __sysctl+0xb0
syscall(2f,2f,2f,0,0) at syscall+0x2a0
Xint0x80_syscall() at Xint0x80_syscall+0x1d
--- syscall (202), eip = 0x280c94ef, esp = 0xbfbfe43c, ebp = 0xbfbfe468 ---
boot() called on cpu#1
evxfevnt-0222 [-112694] AcpiDisable           : ACPI mode disabled

syncing disks, buffers remaining... VOP_LOCK: 0xc7431378 is not locked but
should be
VOP_FSYNC: 0xc7431378 is not locked but should be
ffs_update: 0xc7431378 is not locked but should be
VOP_FSYNC: 0xc7431378 is not locked but should be
VOP_UNLOCK: 0xc7431378 is not locked but should be
panic: pmap_invalidate_range: interrupts disabled
at line 631 in file /usr/src/sys/i386/i386/pmap.ccpuid = 1;
boot() called on cpu#1
Uptime: 19h10m50s
Dumping 1023 MB
 16 32 48 64 80 96 112 128 144 160 176 192 208 224 240 256 272 288 304 320
336 352 368 384 400 416 432 448 464 480 496 512 528 544 560 576 592 608 624
640 656 672 688 704 720 736 752 768 784 800 816 832 848 864 880 896 912 928
944 960 976 992 1008
Dump complete

---------------------------------------
Robin P. Blanchard
Systems Integration Specialist
Georgia Center for Continuing Education
fon: 706.542.2404 < > fax: 706.542.6546
--------------------------------------- 
Received on Fri Feb 06 2004 - 07:27:18 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:37:42 UTC