RE: Processes blocked on ufs or getblk

From: Robin P. Blanchard <robin.blanchard_at_gactr.uga.edu>
Date: Wed, 28 Jan 2004 13:24:11 -0500
Ok...I've just triggered another stuck getblk process. This time it happened
during "make install" of apache2. Below is an output of a basic ddb
investigation. Kernel config and dmesg at:
http://people.gactr.uga.edu/robin/getblk.kernel.txt
http://people.gactr.uga.edu/robin/getblk.dmesg.txt

I will leave the machine in its current locked state should anyone have
anything they'd like for me to try at this point. 

>From /var/log/messages:
...
Jan 28 13:09:00 newweb kernel: Locked vnodes
Jan 28 13:09:00 newweb kernel: 0xc7003378: tag ufs, type VREG, usecount 2,
write
count 0, refcount 1, flags (VV_OBJBUF), lock type ufs: EXCL (count 1) by
thread 0xc6066d20 (pid 81250)
Jan 28 13:09:00 newweb kernel: ino 2523831, on dev aacd0s1g (4, 26)
...

# uptime
 1:11PM  up  3:04, 2 users, load averages: 0.00, 0.00, 0.00

# top |fgrep getblk
81250 root      -4    0  1552K  1100K getblk 0   0:01  0.00%  0.00%
pkg_create

# sysctl debug.enter_debugger=ddb
debug.enter_debuDgger: ebugger("debug.enter_debugger")
Stopped at      Debugger+0x55:  xchgl   %ebx,in_Debugger.0
db> ps
  pid   proc     uarea   uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
81281 c61d5370 ebe97000    0 81278 81281 0004002 [CPU 0] sysctl
81278 c5de1528 e14d2000    0   586 81278 0004002 [SLP]wait 0xc5de1528] bash
81267 c62d5dc0 ebf71000    0   697 81267 0004002 [SLP]ttyin 0xc620b440] bash
81250 c6065c08 ebe30000    0 81125 76764 0004002 [SLP]getblk 0xd289953c]
pkg_create
81125 c62cac08 ebf2a000    0 76764 76764 0004002 [SLP]wait 0xc62cac08] sh
76764 c6350000 ebfe1000    0  1479 76764 0004002 [SLP]wait 0xc6350000] make
 1479 c61686e0 ebe80000    0   697  1479 0004002 [SLP]wait 0xc61686e0] bash
  697 c60161b8 e9bbd000    0   502   697 0000100 [CV]select 0xc06c87e4] sshd
  586 c61d5dc0 ebee5000    0     1   586 0004102 [SLP]wait 0xc61d5dc0] login
  585 c61d8000 ebee6000    0     1   585 0004002 [SLP]ttyin 0xc620ba10] getty
  584 c6168a50 ebe82000    0     1   584 0004002 [SLP]ttyin 0xc5ff9210] getty
  583 c60186e0 e9bed000    0     1   583 0004002 [SLP]ttyin 0xc5d51610] getty
  582 c61d81b8 ebee7000    0     1   582 0004002 [SLP]ttyin 0xc5e57c10] getty
  581 c61d8370 ebee8000    0     1   581 0004002 [SLP]ttyin 0xc5e57410] getty
  525 c6166370 ebe51000    0     1   525 0000000 [SLP]nanslp 0xc069ab1c] cron
  511 c6166a50 ebe55000   25     1   511 0000100 [SLP]pause 0xc61a4000]
sendmail
  508 c6018c08 e9bf0000    0     1   508 0000100 [CV]select 0xc06c87e4]
sendmail
  502 c5de1a50 e14d5000    0     1   502 0000100 [CV]select 0xc06c87e4] sshd
  479 c5de1898 e14d4000    0     1   479 0000000 [CV]select 0xc06c87e4] ntpd
  437 c6166c08 ebe56000    0   431   431 0000100 [CV]select 0xc06c87e4]
rpc.lockd
  431 c6065a50 ebe2f000    0     1   431 0000000 [CV]select 0xc06c87e4]
rpc.lockd
  429 c5de1dc0 e14d7000    0     1   429 0000000 [CV]select 0xc06c87e4]
rpc.statd
  354 c6016000 e9bbc000    1     1   354 0000100 [CV]select 0xc06c87e4]
rpcbind
  335 c60656e0 ebe2d000    0     1   335 0000000 [CV]select 0xc06c87e4]
syslogd
  229 c60651b8 ebe2a000    0     1   229 0000000 [SLP]pause 0xc607b000]
adjkerntz
   81 c60166e0 e9bc0000    0     0     0 0000204 [SLP]- 0xc06d3eac] nfsiod 3
   80 c6016898 e9bc1000    0     0     0 0000204 [SLP]- 0xc06d3ea8] nfsiod 2
   79 c6016a50 e9bc2000    0     0     0 0000204 [SLP]- 0xc06d3ea4] nfsiod 1
   78 c6016c08 e9bc3000    0     0     0 0000204 [SLP]- 0xc06d3ea0] nfsiod 0
   77 c6016dc0 e9bc4000    0     0     0 0000204 [SLP]vlruwt 0xc6016dc0]
vnlru
   76 c6018000 e9bc5000    0     0     0 0000204 [SLP]syncer 0xc069a4e0]
syncer
   75 c5d8fa50 e149f000    0     0     0 0000204 [SLP]psleep 0xc06c8c8c]
bufdaemon
   74 c5d8fc08 e14a0000    0     0     0 000020c [SLP]pgzero 0xc06da6c8]
pagezero
   73 c5d8fdc0 e14a1000    0     0     0 0000204 [SLP]psleep 0xc06da720]
vmdaemon
   72 c5ddf000 e14a2000    0     0     0 0000204 [SLP]psleep 0xc06da70c]
pagedaemon
   71 c5ddf1b8 e14a3000    0     0     0 0000204 [LOCK  Giant c6003b00] swi0:
tty:sio
    9 c5ddf370 e14a4000    0     0     0 0000204 [SLP]aifthd 0xc5ddf370]
aac0aif
    8 c5ddf528 e14a5000    0     0     0 0000204 [SLP]actask 0xc07e042c]
acpi_task2
    7 c5ddf6e0 e14a6000    0     0     0 0000204 [SLP]actask 0xc07e042c]
acpi_taSk1
    6 c5ddf898 e14a7000    0     0     0 0000204 [SLP]actask 0xc07e042c]
acpi_task0
   70 c5ddfa50 e14a8000    0     0     0 0000204 [IWAIT] swi7: task queue
   69 c5ddfc08 e14a9000    0     0     0 0000204 [IWAIT] swi3: cambio
   68 c5ddfdc0 e14aa000    0     0     0 0000204 new [IWAIT] swi2: camnet
   67 c5de1000 e14ab000    0     0     0 0000204 [IWAIT] swi7: acpitaskq
   66 c5de11b8 e14ac000    0     0     0 0000204 new [IWAIT] swi5:+
    5 c5de1370 e14ad000    0     0     0 0000204 [SLP]tqthr 0xc06a2848]
taskqueue
   65 c5d811b8 e1449000    0     0     0 0000204 new [IWAIT] swi6:+
   64 c5d81370 e144a000    0     0     0 0000204 [SLP]- 0xc0692700] random
    4 c5d81528 e144b000    0     0     0 0000204 [SLP]- 0xc06951c0] g_down
    3 c5d816e0 e144c000    0     0     0 0000204 [SLP]- 0xc06951bc] g_up
    2 c5d81898 e144d000    0     0     0 0000204 [SLP]- 0xc06951b4] g_event
   63 c5d81a50 e144e000    0     0     0 0000204 [IWAIT] swi1: net
   62 c5d81c08 e144f000    0     0     0 0000204 new [IWAIT] swi4: vm
   61 c5d81dc0 e1450000    0     0     0 000020c [LOCK  Giant c6003b00] swi8:
tty:sio clock
   60 c5d8f000 e1451000    0     0     0 0000204 new [IWAIT] irq0: clk
   59 c5d8f1b8 e1452000    0     0     0 0000204 new [IWAIT] irq47:
   58 c5d8f370 e1453000    0     0     0 0000204 new [IWAIT] irq46:
   57 c5d8f528 e1454000    0     0     0 0000204 new [IWAIT] irq45:
   56 c5d8f6e0 e1455000    0     0     0 0000204 new [IWAIT] irq44:
   55 c5d8f898 e149e000    0     0     0 0000204 new [IWAIT] irq43:
   54 c5d68a50 e1418000    0     0     0 0000204 new [IWAIT] irq42:
   53 c5d68c08 e1419000    0     0     0 0000204 new [IWAIT] irq41:
   52 c5d68dc0 e141a000    0     0     0 0000204 new [IWAIT] irq40:
   51 c5d7e000 e141b000    0     0     0 0000204 new [IWAIT] irq39:
   50 c5d7e1b8 e141c000    0     0     0 0000204 new [IWAIT] irq38:
   49 c5d7e370 e141d000    0     0     0 0000204 new [IWAIT] irq37:
   48 c5d7e528 e141e000    0     0     0 0000204 new [IWAIT] irq36:
   47 c5d7e6e0 e141f000    0     0     0 0000204 new [IWAIT] irq35:
   46 c5d7e898 e1420000    0     0     0 0000204 new [IWAIT] irq34:
   45 c5d7ea50 e1421000    0     0     0 0000204 new [IWAIT] irq33:
   44 c5d7ec08 e1446000    0     0     0 0000204 new [IWAIT] irq32:
   43 c5d7edc0 e1447000    0     0     0 0000204 new [IWAIT] irq31:
   42 c5d81000 e1448000    0     0     0 0000204 [IWAIT] irq30: aac0
   41 c5d58528 e13e8000    0     0     0 0000204 [LOCK  Giant c6003b00]
irq29: bge1
   40 c5d586e0 e13e9000    0     0     0 0000204 [LOCK  Giant c6003b00]
irq28: bge0
   39 c5d58898 e13ea000    0     0     0 0000204 new [IWAIT] irq27:
   38 c5d58a50 e13eb000    0     0     0 0000204 new [IWAIT] irq26:
   37 c5d58c08 e13ec000    0     0     0 0000204 new [IWAIT] irq25:
   36 c5d58dc0 e13ed000    0     0     0 0000204 new [IWAIT] irq24:
   35 c5d68000 e13ee000    0     0     0 0000204 new [IWAIT] irq23:
   34 c5d681b8 e1413000    0     0     0 0000204 new [IWAIT] irq22:
   33 c5d68370 e1414000    0     0     0 0000204 new [IWAIT] irq21:
   32 c5d68528 e1415000    0     0     0 0000204 new [IWAIT] irq20:
   31 c5d686e0 e1416000    0     0     0 0000204 new [IWAIT] irq19:
   30 c5d68898 e1417000    0     0     0 0000204 new [IWAIT] irq18:
   29 c1e691b8 df9b0000    0     0     0 0000204 new [IWAIT] irq17:
   28 c1e69370 df9b1000    0     0     0 0000204 new [IWAIT] irq16:
   27 c1e69528 df9b2000    0     0     0 0000204 new [IWAIT] irq15: ata1
   26 c1e696e0 df9b3000    0     0     0 0000204 [IWAIT] irq14: ata0
   25 c1e69898 df9d8000    0     0     0 0000204 new [IWAIT] irq13:
   24 c1e69a50 df9d9000    0     0     0 0000204 new [IWAIT] irq12: psm0
   23 c1e69c08 df9da000    0     0     0 0000204 new [IWAIT] irq11:
   22 c1e69dc0 df9db000    0     0     0 0000204 new [IWAIT] irq10:
   21 c5d58000 e13e5000    0     0     0 0000204 new [IWAIT] irq9: acpi0
   20 c5d581b8 e13e6000    0     0     0 0000204 new [IWAIT] irq8: rtc
   19 c5d58370 e13e7000    0     0     0 0000204 new [IWAIT] irq7:
   18 c1e60000 df95e000    0     0     0 0000204 [IWAIT] irq6: fdc0
   17 c1e601b8 df9a7000    0     0     0 0000204 new [IWAIT] irq5:
   16 c1e60370 df9a8000    0     0     0 0000204 new [IWAIT] irq4: sio0
   15 c1e60528 df9a9000    0     0     0 0000204 new [IWAIT] irq3: sio1
   14 c1e606e0 df9aa000    0     0     0 0000204 new [IWAIT] irq2:
   13 c1e60898 df9ab000    0     0     0 0000204 [IWAIT] irq1: atkbd0
   12 c1e60a50 df9ac000    0     0     0 000020c [Can run] idle: cpu0
   11 c1e60c08 df9ad000    0     0     0 000020c [CPU 1] idle: cpu1
    1 c1e60dc0 df9ae000    0     0     1 0004200 [SLP]wait 0xc1e60dc0] init
   10 c1e69000 df9af000    0     0     0 0000204 [CV]ktrace 0xc0698704]
ktrace
    0 c0695260 c081f000    0     0     0 0000200 [SLP]sched 0xc0695260]
swapper
db> show lockedvnods
Locked vnodes
0xc7003378: tag ufs, type VREG, usecount 2, writecount 0, refcount 1, flags
(VV_OBJBUF), lock type ufs: EXCL (count 1) by thread 0xc6066d20 (pid 81250)
        ino 2523831, on dev aacd0s1g (4, 26)
db> tr 81250
sched_switch(c6066d20,2,c0649d3a,1d2,f8daead9) at sched_switch+0x155
mi_switch(1,50,c0649d3a,ca,1) at mi_switch+0x298
msleep(d289953c,c0699074,50,c065007c,0) at msleep+0x500
acquire(ebe02a34,2000020,600,e7,c064fd03) at acquire+0xae
debuglockmgr(d289953c,2090022,c7003378,c6066d20,c0643f29) at
debuglockmgr+0x477
getblk(c7003378,0,0,3000,0) at getblk+0x1a7
breadn(c7003378,0,0,3000,0) at breadn+0x52
bread(c7003378,0,0,3000,0) at bread+0x4c
ffs_read(ebe02be4,c0651cf6,c6066d20,c0651cd8,20e) at ffs_read+0x44f
vn_read(c60598c4,ebe02c80,c62d3180,0,c6066d20) at vn_read+0x215
dofileread(c6066d20,c60598c4,4,bfbfd800,400) at dofileread+0xdc
read(c6066d20,ebe02d14,c0660726,3f2,3) at read+0x6b
syscall(2f,2f,bfbf002f,0,2bdb) at syscall+0x2c0
Xint0x80_syscall() at Xint0x80_syscall+0x1d
--- syscall (3, FreeBSD ELF32, read), eip = 0x280da15f, esp = 0xbfbfd6fc, ebp
=0xbfbfdc18 ---







---------------------------------------
Robin P. Blanchard
Systems Integration Specialist
Georgia Center for Continuing Education
fon: 706.542.2404 < > fax: 706.542.6546
---------------------------------------
 
Received on Wed Jan 28 2004 - 09:24:22 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:37:40 UTC