-current kernel panic

From: Thierry Herbelot <thierry_at_herbelot.com>
Date: Mon, 17 Dec 2007 19:35:09 +0100
Hello,

for some days, I see panics on my oldish SMP up-to-date -current box 
(FreeBSD 8.0-CURRENT #1984: Mon Dec 17 09:54:07 CET 2007)

the last working kernel was built on Thu Dec 13 09:58:39 CET 2007.
the kernel config is a straight GENERIC (incl. all debug options).

The panic always occurs at the beginning of the make installworld.
all partitions except root are hosted on a ZFS mirror. the root partition is 
hosted on a geom mirror.
# mount
/dev/mirror/gm0a on / (ufs, local, soft-updates)
devfs on /dev (devfs, local)
tank on /tank (zfs, local, noatime)
tank/files1 on /tank/files1 (zfs, local, noatime)
tank/files3 on /tank/files3 (zfs, local, noatime)
tank/tmp on /tmp (zfs, local, noatime)
tank/usr on /usr (zfs, local, noatime)
tank/var on /var (zfs, local, noatime)

	TfH

the panic message is :
panic: Assertion td->td_lock == TDQ_LOCKPTR(tdq) failed 
at /tank/files1/src/sys/kern/sched_ule.c:1858
cpuid = 0
KDB: enter: panic
[thread pid 11 tid 100003 ]
Stopped at      kdb_enter+0x32: leave

more DDB traces for this panic :
db> where
Tracing pid 11 tid 100003 td 0xc2763880
kdb_enter(c0aff8da,0,c0afaad4,dd238a2c,0,...) at kdb_enter+0x32
panic(c0afaad4,c0b018c6,c0b01376,742,c0c1a680,...) at panic+0x124
sched_switch(c2763880,0,1,17b,86a5c2a4,...) at sched_switch+0xe6
mi_switch(1,0,c0b03bf0,2e2,c2736e60,...) at mi_switch+0x217
turnstile_wait(c2736e60,c36dbaa0,0,184,c1474708,...) at turnstile_wait+0x4cb
_mtx_lock_sleep(c1474708,c2763880,0,c0b213d7,726,...) at _mtx_lock_sleep+0x18e
_mtx_lock_flags(c1474708,0,c0b213d7,726,c0b04452,...) at _mtx_lock_flags+0xef
uma_zalloc_arg(c1472960,0,2,2,dd238b9e,...) at uma_zalloc_arg+0xd3
malloc(2,c0bb1a40,2,131,208bb0,...) at malloc+0xd2
getenv(c0b2dfe8,c0c20a80,c2763880,3,dd238cc8,...) at getenv+0xa6
cpu_idle_default(dd238cf8,c077d839,c0c20a80,0,c0b01376,...) at 
cpu_idle_default+0x13
cpu_idle(c0c20a80,0,c0b01376,322,c0c1a680,...) at cpu_idle+0x28
sched_idletd(0,dd238d38,c0afc024,30c,c2761804,...) at sched_idletd+0x249
fork_exit(c077d5f0,0,dd238d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdd238d70, ebp = 0 ---
db> show allpcpu
Current CPU: 0

cpuid        = 0
curthread    = 0xc2763880: pid 11 "idle: cpu0"
curpcb       = 0xdd238d90
fpcurthread  = none
idlethread   = 0xc2763880: pid 11 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc2ba7440: pid 89 "spa_zio_intr_2"
curpcb       = 0xdee70d90
fpcurthread  = none
idlethread   = 0xc2763aa0: pid 11 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

db> ps
  pid  ppid  pgrp   uid   state   wmesg     wchan    cmd
  272     0     0     0  SL      spa->spa 0xc29c2460 [solthread 0xc2a6937]
  268   265   112     0  R+                          make
  265   264   112     0  S+      wait     0xc2bba2ac sh
  264   262   112     0  S+      wait     0xc2b88ab0 make
  262   257   112     0  S+      wait     0xc3da22ac sh
  257   256   112     0  S+      wait     0xc3dec000 make
  256   209   112     0  S+      wait     0xc2bba804 sh
  209   208   112     0  S+      wait     0xc29ffab0 make
  208   119   112     0  S+      wait     0xc3dec804 sh
  180     0     0     0  SL      vgeom:io 0xc29be148 [vdev:worker ad2s2d]
  179     0     0     0  SL      vgeom:io 0xc2e3c288 [vdev:worker ad0s2d]
  119   117   112     0  S+      wait     0xc2ba3804 make
  117   112   112     0  S+      wait     0xc2bba000 sh
  112    29   112     0  S+      wait     0xc2bbaab0 make
  106     0     0     0  SL      tx->tx_t 0xc2de372c [txg_thread_enter]
  105     0     0     0  SL      zio->io_ 0xc2d2aab8 [txg_thread_enter]
  104     0     0     0  SL      tx->tx_q 0xc2de371c [txg_thread_enter]
  101     0     0     0  SL      tq->tq_d 0xc2af45e4 [spa_zio_intr_5]
  100     0     0     0  SL      tq->tq_d 0xc2af45e4 [spa_zio_intr_5]
   99     0     0     0  SL      tq->tq_d 0xc2af46b0 [spa_zio_issue_5]
   98     0     0     0  SL      tq->tq_d 0xc2af46b0 [spa_zio_issue_5]
   97     0     0     0  SL      tq->tq_d 0xc2af477c [spa_zio_intr_4]
   96     0     0     0  SL      tq->tq_d 0xc2af477c [spa_zio_intr_4]
   95     0     0     0  SL      tq->tq_d 0xc2af4848 [spa_zio_issue_4]
   94     0     0     0  SL      tq->tq_d 0xc2af4848 [spa_zio_issue_4]
   93     0     0     0  SL      tq->tq_d 0xc2af4914 [spa_zio_intr_3]
   92     0     0     0  SL      tq->tq_d 0xc2af4914 [spa_zio_intr_3]
   91     0     0     0  SL      tq->tq_d 0xc2af49e0 [spa_zio_issue_3]
   90     0     0     0  SL      tq->tq_d 0xc2af49e0 [spa_zio_issue_3]
   89     0     0     0  RL      CPU 1               [spa_zio_intr_2]
   88     0     0     0  SL      tq->tq_d 0xc2af4aac [spa_zio_intr_2]
   87     0     0     0  SL      tq->tq_d 0xc2af4b78 [spa_zio_issue_2]
   86     0     0     0  SL      tq->tq_d 0xc2af4b78 [spa_zio_issue_2]
   85     0     0     0  SL      tq->tq_d 0xc2af4c44 [spa_zio_intr_1]
   84     0     0     0  SL      tq->tq_d 0xc2af4c44 [spa_zio_intr_1]
   83     0     0     0  SL      tq->tq_d 0xc2af4d10 [spa_zio_issue_1]
   82     0     0     0  SL      tq->tq_d 0xc2af4d10 [spa_zio_issue_1]
   81     0     0     0  SL      tq->tq_d 0xc2af4ddc [spa_zio_intr_0]
   80     0     0     0  SL      tq->tq_d 0xc2af4ddc [spa_zio_intr_0]
   79     0     0     0  SL      tq->tq_d 0xc2af4ea8 [spa_zio_issue_0]
   78     0     0     0  SL      tq->tq_d 0xc2af4ea8 [spa_zio_issue_0]
   50     0     0     0  SL      arc_recl 0xc2ab5e0c [arc_reclaim_thread]
   48     0     0     0  SL      tq->tq_d 0xc2af5050 [system_taskq]
   47     0     0     0  SL      tq->tq_d 0xc2af5050 [system_taskq]
   29     1    29     0  Ss+     wait     0xc2a00ab0 sh
   28     0     0     0  SL      sdflush  0xc0c75ec4 [softdepflush]
   27     0     0     0  SL      vlruwt   0xc2864ab0 [vnlru]
   26     0     0     0  SL      tx->tx_s 0xc2de3714 [syncer]
   25     0     0     0  SL      psleep   0xc0c6a644 [bufdaemon]
   24     0     0     0  SL      pgzero   0xc0c76a80 [pagezero]
   23     0     0     0  SL      psleep   0xc0c76698 [vmdaemon]
   22     0     0     0  SL      psleep   0xc0c76660 [pagedaemon]
   21     0     0     0  SL      m:w1     0xc28db000 [g_mirror gm0]
    9     0     0     0  SL      waiting_ 0xc0c6c3cc [sctp_iterator]
    8     0     0     0  SL      -        0xc287ae3c [fdc0]
   20     0     0     0  SL      usbevt   0xc2851a10 [usb4]
   19     0     0     0  SL      usbevt   0xc2872210 [usb3]
   18     0     0     0  SL      usbevt   0xc285c210 [usb2]
   17     0     0     0  SL      usbevt   0xc285a210 [usb1]
   16     0     0     0  SL      usbtsk   0xc0c16354 [usbtask-dr]
   15     0     0     0  SL      usbtsk   0xc0c16340 [usbtask-hc]
   14     0     0     0  SL      usbevt   0xc282c210 [usb0]
    7     0     0     0  SL      -        0xc2789d00 [kqueue taskq]
    6     0     0     0  SL      ccb_scan 0xc0be7514 [xpt_thrd]
    5     0     0     0  SL      -        0xc27f7200 [thread taskq]
   13     0     0     0  SL      -        0xc0c18a54 [yarrow]
    4     0     0     0  SL      -        0xc0c16aac [g_down]
    3     0     0     0  SL      -        0xc0c16aa8 [g_up]
    2     0     0     0  SL      -        0xc0c16aa0 [g_event]
   12     0     0     0  WL      (threaded)          intr
100036                   I                           [irq5: pcm1]
100035                   I                           [swi0: sio]
100034                   I                           [irq7: ppc0]
100032                   I                           [irq12: psm0]
100031                   I                           [irq1: atkbd0]
100029                   I                           [irq18: ehci0++]
100027                   I                           [irq17: ohci2]
100025                   I                           [irq16: ohci1]
100020                   I                           [irq19: pcm0 dc0++]
100019                   I                           [irq15: ata1]
100018                   I                           [irq14: ata0]
100017                   I                           [swi5: +]
100015                   I                           [swi2: cambio]
100013                   I                           [swi6: task queue]
100012                   I                           [swi6: Giant taskq]
100006                   I                           [swi3: vm]
100005                   I                           [swi4: clock sio]
100004                   I                           [swi1: net]
   11     0     0     0  RL      (threaded)          idle
100003                   L      *16       0xc2736e60 [idle: cpu0]
100002                   CanRun                      [idle: cpu1]
    1     0     1     0  SLs     wait     0xc2761ab0 [init]
   10     0     0     0  SL      audit_wo 0xc0c75930 [audit]
    0     0     0     0  WLs                         [swapper]
db>  

ANOTHER backtrace (with a previous kernel) was :
 /etc/rc.d/zfs start
WARNING: ZFS is considered to be an experimental feature in FreeBSD.
ZFS WARNING: Recommended minimum RAM size is 512MB; expect unstable behavior.
ZFS filesystem version 6
ZFS storage pool version 6
panic: Assertion td->td_lock == TDQ_LOCKPTR(tdq) failed 
at /tank/files1/src/sys/kern/sched_ule.c:1858
cpuid = 1
KDB: enter: panic
[thread pid 11 tid 100002 ]
Stopped at      kdb_enter+0x32: leave
db> where
Tracing pid 11 tid 100002 td 0xc2763aa0
kdb_enter(c0aafada,1,c0aaacd4,dd235a2c,1,...) at kdb_enter+0x32
panic(c0aaacd4,c0ab17e1,c0ab1576,742,c0bc9e40,...) at panic+0x124
sched_switch(c2763aa0,0,1,17b,cea602ac,...) at sched_switch+0xe6
mi_switch(1,0,c0ab3df0,2e2,c2736e10,...) at mi_switch+0x217
turnstile_wait(c2736e10,c2ba1aa0,0,192,c1474708,...) at turnstile_wait+0x4cb
_mtx_lock_sleep(c1474708,c2763aa0,0,c0ad15b9,726,...) at _mtx_lock_sleep+0x18e
_mtx_lock_flags(c1474708,0,c0ad15b9,726,c0ab4652,...) at _mtx_lock_flags+0xef
uma_zalloc_arg(c1472960,0,2,2,dd235b9e,...) at uma_zalloc_arg+0xd3
malloc(2,c0b61aa0,2,131,205bb8,...) at malloc+0xd2
getenv(c0ade187,c0ade19d,c2763aa0,3,dd235cc8,...) at getenv+0xa6
cpu_idle_default(dd235cf8,c077c409,c0bcfc80,0,c0ab1576,...) at 
cpu_idle_default+0x13
cpu_idle(c0bcfc80,0,c0ab1576,322,c0bc9e40,...) at cpu_idle+0x28
sched_idletd(0,dd235d38,c0aac224,30c,c2761804,...) at sched_idletd+0x249
fork_exit(c077c1c0,0,dd235d38) at fork_exit+0xb8
fork_trampoline() at fork_trampoline+0x8
--- trap 0, eip = 0, esp = 0xdd235d70, ebp = 0 ---
db>
Received on Mon Dec 17 2007 - 17:35:22 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:39:24 UTC