Giant deadlock related to twe

From: Doug White <dwhite_at_gumbysoft.com>
Date: Sun, 22 Aug 2004 15:32:35 -0700 (PDT)
Hey folks,

Just got this on my amd64 box. A disk flaked out in my machine, which has
a 3ware 8006-2LP with 2 80GB drives in a RAID0.  My X session locked up
and was able to break to ddb.  Some ddb twiddling follows.  It looks like,
at first glance, some sort of deadlock against softupdates.

twed0: drive timeouttwed0: drive timeouttwed0: drive timeouttwed0: drive
timeout
twed0: drive timeouttwed0: drive timeouttwed0: drive timeouttwe0: AEN:
<twe0: drive timeout for unknown unit 1>
twed0: controller error - device failure (flags = 0x80)
twed0: controller error - device failure (flags = 0x80)
twed0: controller error - device failure (flags = 0x80)
twed0: controller error - device failure (flags = 0x80)
KDB: enter: Line break on console
[thread 100082]
Stopped at      kdb_enter+0x31: leave
db> tr   # NOTE: this is the syslog process on CPU0, pid 304
kdb_enter() at kdb_enter+0x31
siointr1() at siointr1+0xe3
siointr() at siointr+0x78
intr_execute_handlers() at intr_execute_handlers+0xb2
lapic_handle_intr() at lapic_handle_intr+0x38
Xapic_isr1() at Xapic_isr1+0x7d
--- interrupt, rip = 0xffffffff8034dd9a, rsp = 0xffffffffb23f3960, rbp =
0xffffffffb23f39b0 ---
inodedep_lookup() at inodedep_lookup+0x7a
flush_inodedep_deps() at flush_inodedep_deps+0x5c
softdep_sync_metadata() at softdep_sync_metadata+0x9f
ffs_fsync() at ffs_fsync+0x44c
fsync() at fsync+0x182
syscall() at syscall+0x374
Xfast_syscall() at Xfast_syscall+0xa8
--- syscall (95, FreeBSD ELF64, fsync), rip = 0x20079c0c0, rsp =
0x7fffffffe808, rbp = 0x50f080 ---
db> ps
  pid   proc     uarea   uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
  811 ffffff00530dab80 ffffffffb48e9000    0   809   800 0004002 [SLPQ
ttyin 0xffffff00009b8410][SLP] less
  810 ffffff005313c2e0 ffffffffb48eb000    0   809   800 0004002 [SLPQ
pipdwt 0xffffff00527cc5a0][SLP] zcat
  809 ffffff005c24a2e0 ffffffffb4814000    0   800   800 0004002 [SLPQ
wait 0xffffff005c24a2e0][SLP] sh
  800 ffffff00626082e0 ffffffffb4765000    0   645   800 0004002 [SLPQ
wait 0xffffff00626082e0][SLP] man
  783 ffffff0056d152e0 ffffffffb4823000 1001   653   783 0004002 [SLPQ
ttyin 0xffffff007ace8c10][SLP] less
  744 ffffff00572fd000 ffffffffb4886000 1001   632   744 0004002 [SLPQ
ttyin 0xffffff0000eaf010][SLP] tcsh
  740 ffffff0056d155c0 ffffffffb4824000 1001     1   594 0004001 [SLPQ
select 0xffffffff80583550][SLP] gconfd-2
  738 ffffff0056d15b80 ffffffffb4826000 1001   733   594 000c000
(threaded)  mozilla-bin
   thread 0xffffff0051e40000 ksegrp 0xffffff005fc80270 [SLPQ kserel
0xffffff005fc80310][SLP]
   thread 0xffffff0053001940 ksegrp 0xffffff005fc80270 [SLPQ kserel
0xffffff005fc80310][SLP]
   thread 0xffffff00530016f0 ksegrp 0xffffff005fc80270 [SLPQ select
0xffffffff80583550][SLP]
   thread 0xffffff0053001250 ksegrp 0xffffff005fc80270 [SLPQ select
0xffffffff80583550][SLP]
   thread 0xffffff005ff05250 ksegrp 0xffffff005c46e1a0 [SLPQ ksesigwait
0xffffff0056d15d30][SLP]
  733 ffffff005c30f5c0 ffffffffb481f000 1001   594   594 0004000 [SLPQ
wait 0xffffff005c30f5c0][SLP] sh
  700 ffffff0056fd85c0 ffffffffb4883000 1001   651   700 0004002 [SLPQ
ttyin 0xffffff0000eae810][SLP] tcsh
  653 ffffff005c24ab80 ffffffffb4817000 1001   651   653 0004002 [SLPQ
pause 0xffffff005c24abf0][SLP] tcsh
  651 ffffff005fe1f2e0 ffffffffb476a000 1001   594   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  645 ffffff005c24a5c0 ffffffffb4815000    0   644   645 0004002 [SLPQ
pause 0xffffff005c24a630][SLP] tcsh
  644 ffffff005c24a8a0 ffffffffb4816000 1001   634   644 0004102 [SLPQ
wait 0xffffff005c24a8a0][SLP] su
  634 ffffff0060029000 ffffffffb47cd000 1001   632   634 0004002 [SLPQ
pause 0xffffff0060029070][SLP] tcsh
  632 ffffff005c54d2e0 ffffffffb4819000 1001   594   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  630 ffffff005fdc88a0 ffffffffb47cb000 1001     1   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  627 ffffff0060029b80 ffffffffb47d1000 1001   594   594 0000000 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  626 ffffff005bfc92e0 ffffffffb47d3000 1001     1   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  624 ffffff005bfc9000 ffffffffb47d2000 1001     1   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  622 ffffff007b4b18a0 ffffffffb2400000 1001     1   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  619 ffffff005bfc95c0 ffffffffb47d4000 1001   594   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  617 ffffff005c54db80 ffffffffb481c000 1001     1   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  615 ffffff005c24a000 ffffffffb47d7000 1001   575   575 0004000 [SLPQ
nanslp 0xffffffff8057ae80][SLP] kwrapper
  603 ffffff005c54d000 ffffffffb4818000 1001     1   594 0000001 [LOCK
Giant ffffff00581b6400] kdeinit
  600 ffffff005c54d8a0 ffffffffb481b000 1001   594   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  597 ffffff005c30f000 ffffffffb481d000 1001     1   594 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  594 ffffff007b4cd000 ffffffffb23b7000 1001     1   594 0000000 [SLPQ
select 0xffffffff80583550][SLP] kdeinit
  575 ffffff007b7288a0 ffffffffb1ac7000 1001   559   575 0004000 [SLPQ
wait 0xffffff007b7288a0][SLP] sh
  559 ffffff00627425c0 ffffffffb4761000    0   556   556 0000000 [SLPQ
wait 0xffffff00627425c0][SLP] kdm
  558 ffffff005fe1fb80 ffffffffb476d000    0   556   558 0004000 [SLPQ
select 0xffffffff80583550][SLP] XFree86
  556 ffffff0062742b80 ffffffffb4763000    0     1   556 0000001 [SLPQ
select 0xffffffff80583550][SLP] kdm
  554 ffffff005fdc8000 ffffffffb47c8000    0     1   554 0004002 [SLPQ
ttyin 0xffffff0000ead410][SLP] getty
  553 ffffff007b4b1b80 ffffffffb2401000    0     1   553 0004002 [SLPQ
ttyin 0xffffff0000a9e010][SLP] getty
  552 ffffff005fdc82e0 ffffffffb47c9000    0     1   552 0004002 [SLPQ
ttyin 0xffffff0000a5cc10][SLP] getty
  551 ffffff005fdc85c0 ffffffffb47ca000    0     1   551 0004002 [SLPQ
ttyin 0xffffff0000a9d410][SLP] getty
  550 ffffff0062608000 ffffffffb4764000    0     1   550 0004002 [SLPQ
ttyin 0xffffff0000a5c010][SLP] getty
  549 ffffff0062742000 ffffffffb4705000    0     1   549 0004002 [SLPQ
ttyin 0xffffff0000a5c810][SLP] getty
  548 ffffff00627422e0 ffffffffb4760000    0     1   548 0004002 [SLPQ
ttyin 0xffffff0000a71410][SLP] getty
  547 ffffff007b4ce000 ffffffffb23bc000    0     1   547 0004002 [SLPQ
ttyin 0xffffff0000a71810][SLP] getty
  546 ffffff00600298a0 ffffffffb47d0000    0     1   546 0004002 [SLPQ
ttyin 0xffffff0000a71c10][SLP] getty
  484 ffffff005fe1f5c0 ffffffffb476b000    0     1   484 0000000 [SLPQ
nanslp 0xffffffff8057ae80][SLP] cron
  470 ffffff005fe1f8a0 ffffffffb476c000   25     1   470 0000100 [SLPQ
pause 0xffffff005fe1f910][SLP] sendmail
  467 ffffff005fe1f000 ffffffffb4769000    0     1   467 0000100 [SLPQ
select 0xffffffff80583550][SLP] sendmail
  461 ffffff007b4cd8a0 ffffffffb23ba000    0     1   461 0000100 [SLPQ
select 0xffffffff80583550][SLP] sshd
  454 ffffff007b4cd5c0 ffffffffb23b9000    0   440   440 0000000 [SLPQ
pause 0xffffff007b4cd630][SLP] ntpd
  440 ffffff00626085c0 ffffffffb4766000    0     1   440 0000000 [SLPQ
select 0xffffffff80583550][SLP] ntpd
  408 ffffff007b4cd2e0 ffffffffb23b8000    0     1   408 0000000 [SLPQ
select 0xffffffff80583550][SLP] usbd
  406 ffffff007b4b12e0 ffffffffb23fe000    0     1   406 0000000 [SLPQ
select 0xffffffff80583550][SLP] moused
  319 ffffff007b4b15c0 ffffffffb23ff000    0     1   319 0000000 [SLPQ
select 0xffffffff80583550][SLP] rpcbind
  304 ffffff007b4ce2e0 ffffffffb23f9000    0     1   304 0000000 [CPU 1]
syslogd
  243 ffffff00626088a0 ffffffffb4767000    0     1   243 0000000 [SLPQ
select 0xffffffff80583550][SLP] dhclient
  189 ffffff007b728b80 ffffffffb1ac8000    0     1   189 0000000 [SLPQ
pause 0xffffff007b728bf0][SLP] adjkerntz
   67 ffffff007b4ce5c0 ffffffffb23fa000    0     0     0 0000204 [SLPQ -
0xffffffffb23eec0c][SLP] schedcpu
   66 ffffff007b4ce8a0 ffffffffb23fb000    0     0     0 0000204 [SLPQ -
0xffffffff8058ce38][SLP] nfsiod 3
   65 ffffff007b4ceb80 ffffffffb23fc000    0     0     0 0000204 [SLPQ -
0xffffffff8058ce30][SLP] nfsiod 2
   64 ffffff007b4b1000 ffffffffb23fd000    0     0     0 0000204 [SLPQ -
0xffffffff8058ce28][SLP] nfsiod 1
   63 ffffff007b7552e0 ffffffffb1a7f000    0     0     0 0000204 [SLPQ -
0xffffffff8058ce20][SLP] nfsiod 0
   62 ffffff007b7555c0 ffffffffb1a80000    0     0     0 0000204 [SLPQ
vlruwt 0xffffff007b7555c0][SLP] vnlru
   61 ffffff007b7558a0 ffffffffb1a81000    0     0     0 0000204 [SLPQ
syncer 0xffffffff8057ab80][SLP] syncer
   60 ffffff007b755b80 ffffffffb1a82000    0     0     0 0000204 [SLPQ
psleep 0xffffffff80583e7c][SLP] bufdaemon
   59 ffffff007b6e5000 ffffffffb1a83000    0     0     0 000020c [SLPQ
pgzero 0xffffffff805957f0][SLP] pagezero
   58 ffffff007b6e52e0 ffffffffb1a84000    0     0     0 0000204 [SLPQ
psleep 0xffffffff8059585c][SLP] vmdaemon
   57 ffffff007b6e55c0 ffffffffb1a85000    0     0     0 0000204 [SLPQ
psleep 0xffffffff8059580c][SLP] pagedaemon
   56 ffffff007b6e58a0 ffffffffb1ac2000    0     0     0 0000204 [IWAIT]
swi0: sio
   55 ffffff007b6e5b80 ffffffffb1ac3000    0     0     0 0000204 [SLPQ -
0xffffff00009ce248][SLP] fdc0
   54 ffffff007b728000 ffffffffb1ac4000    0     0     0 0000204 [SLPQ
usbevt 0xffffffff80823420][SLP] usb1
   53 ffffff007b7282e0 ffffffffb1ac5000    0     0     0 0000204 [SLPQ
usbtsk 0xffffffff8056e4f0][SLP] usbtask
   52 ffffff007b7285c0 ffffffffb1ac6000    0     0     0 0000204 [SLPQ
usbevt 0xffffffff80821420][SLP] usb0
    9 ffffff007b751000 ffffffffb1a38000    0     0     0 0000204 [SLPQ
actask 0xffffffff8056bd00][SLP] acpi_task2
    8 ffffff007b7512e0 ffffffffb1a39000    0     0     0 0000204 [SLPQ
actask 0xffffffff8056bd00][SLP] acpi_task1
    7 ffffff007b7515c0 ffffffffb1a3a000    0     0     0 0000204 [SLPQ
actask 0xffffffff8056bd00][SLP] acpi_task0
   51 ffffff007b7518a0 ffffffffb1a3b000    0     0     0 0000204 [IWAIT]
swi6: task queue
    6 ffffff007b751b80 ffffffffb1a3c000    0     0     0 0000204 [SLPQ -
0xffffff000099ae00][SLP] kqueue taskq
   50 ffffff007b752000 ffffffffb1a3d000    0     0     0 0000204 [IWAIT]
swi6: acpitaskq
   49 ffffff007b7522e0 ffffffffb1a7a000    0     0     0 0000204 [IWAIT]
swi6:+
    5 ffffff007b7525c0 ffffffffb1a7b000    0     0     0 0000204 [SLPQ -
0xffffff00008c3a80][SLP] thread taskq
   48 ffffff007b7528a0 ffffffffb1a7c000    0     0     0 0000204 [IWAIT]
swi6:+
   47 ffffff007b752b80 ffffffffb1a7d000    0     0     0 0000204 [SLPQ -
0xffffffff8056c1c0][SLP] yarrow
    4 ffffff007b755000 ffffffffb1a7e000    0     0     0 0000204 [SLPQ -
0xffffffff80571ea8][SLP] g_down
    3 ffffff007b700000 ffffffffb19f2000    0     0     0 0000204 [SLPQ -
0xffffffff80571ea0][SLP] g_up
    2 ffffff007b7002e0 ffffffffb19f3000    0     0     0 0000204 [SLPQ -
0xffffffff80571e90][SLP] g_event
   46 ffffff007b7005c0 ffffffffb19f4000    0     0     0 0000204 [IWAIT]
swi1: net
   45 ffffff007b7008a0 ffffffffb19f5000    0     0     0 0000204 [IWAIT]
swi4: vm
   44 ffffff007b700b80 ffffffffb19f6000    0     0     0 000020c [LOCK
Giant ffffff00581b6400] swi5: clock sio
   43 ffffff007b704000 ffffffffb1a33000    0     0     0 0000204 [IWAIT]
irq31:
   42 ffffff007b7042e0 ffffffffb1a34000    0     0     0 0000204 [IWAIT]
irq30:
   41 ffffff007b7045c0 ffffffffb1a35000    0     0     0 0000204 [LOCK
Giant ffffff00581b6400] irq29: twe0
   40 ffffff007b7048a0 ffffffffb1a36000    0     0     0 0000204 [IWAIT]
irq28:
   39 ffffff007b704b80 ffffffffb1a37000    0     0     0 0000204 [IWAIT]
irq27: bge0 bge1
   38 ffffff007b7262e0 ffffffffb19ad000    0     0     0 0000204 [IWAIT]
irq26:
   37 ffffff007b7265c0 ffffffffb19ae000    0     0     0 0000204 [IWAIT]
irq25:
   36 ffffff007b7268a0 ffffffffb19af000    0     0     0 0000204 [IWAIT]
irq24:
   35 ffffff007b726b80 ffffffffb19b0000    0     0     0 0000204 [IWAIT]
irq23:
   34 ffffff007b754000 ffffffffb19ed000    0     0     0 0000204 [IWAIT]
irq22:
   33 ffffff007b7542e0 ffffffffb19ee000    0     0     0 0000204 [IWAIT]
irq21:
   32 ffffff007b7545c0 ffffffffb19ef000    0     0     0 0000204 [IWAIT]
irq20:
   31 ffffff007b7548a0 ffffffffb19f0000    0     0     0 0000204 [LOCK
Giant ffffff00581b6400] irq19: ohci0 ohci1
   30 ffffff007b754b80 ffffffffb19f1000    0     0     0 0000204 [IWAIT]
irq18:
   29 ffffff007b77c8a0 ffffffffb1987000    0     0     0 0000204 [IWAIT]
irq17:
   28 ffffff007b77cb80 ffffffffb1988000    0     0     0 0000204 [IWAIT]
irq16:
   27 ffffff007b743000 ffffffffb1989000    0     0     0 0000204 [IWAIT]
irq15: ata1
   26 ffffff007b7432e0 ffffffffb19a8000    0     0     0 0000204 [IWAIT]
irq14: ata0
   25 ffffff007b7435c0 ffffffffb19a9000    0     0     0 0000204 [IWAIT]
irq13:
   24 ffffff007b7438a0 ffffffffb19aa000    0     0     0 0000204 [IWAIT]
irq12:
   23 ffffff007b743b80 ffffffffb19ab000    0     0     0 0000204 [IWAIT]
irq11:
   22 ffffff007b726000 ffffffffb19ac000    0     0     0 0000204 [IWAIT]
irq10:
   21 ffffff007b77a2e0 ffffffffb1962000    0     0     0 0000204 [IWAIT]
irq9: acpi0
   20 ffffff007b77a5c0 ffffffffb1963000    0     0     0 0000204 [IWAIT]
irq8: rtc
   19 ffffff007b77a8a0 ffffffffb1982000    0     0     0 0000204 [IWAIT]
irq7:
   18 ffffff007b77ab80 ffffffffb1983000    0     0     0 0000204 [IWAIT]
irq6: fdc0
   17 ffffff007b77c000 ffffffffb1984000    0     0     0 0000204 [IWAIT]
irq5:
   16 ffffff007b77c2e0 ffffffffb1985000    0     0     0 0000204 [IWAIT]
irq4: sio0
   15 ffffff007b77c5c0 ffffffffb1986000    0     0     0 0000204 [IWAIT]
irq3: sio1
   14 ffffff007b771000 ffffffffb1920000    0     0     0 0000204 [IWAIT]
irq0: clk
   13 ffffff007b7712e0 ffffffffb195d000    0     0     0 0000204 [LOCK
Giant ffffff00581b6400] irq1: atkbd0
   12 ffffff007b7715c0 ffffffffb195e000    0     0     0 000020c [CPU 0]
idle: cpu0
   11 ffffff007b7718a0 ffffffffb195f000    0     0     0 000020c [Can run]
idle: cpu1
    1 ffffff007b771b80 ffffffffb1960000    0     0     1 0004200 [SLPQ
wait 0xffffff007b771b80][SLP] init
   10 ffffff007b77a000 ffffffffb1961000    0     0     0 0000204 [SLPQ
ktrace 0xffffffff805777d0][SLP] ktrace
    0 ffffffff80572020 ffffffff8065e000    0     0     0 0000200 [SLPQ
sched 0xffffffff80572020][SLP] swapper
db> tr 41
sched_switch() at sched_switch+0xc8
mi_switch() at mi_switch+0x1b0
turnstile_wait() at turnstile_wait+0x37b
_mtx_lock_sleep() at _mtx_lock_sleep+0x149
ithread_loop() at ithread_loop+0x1b2
fork_exit() at fork_exit+0xab
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip = 0, rsp = 0xffffffffb1a14d00, rbp = 0 ---



-- 
Doug White                    |  FreeBSD: The Power to Serve
dwhite_at_gumbysoft.com          |  www.FreeBSD.org
Received on Sun Aug 22 2004 - 20:32:35 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:38:07 UTC