Re: ZFS Crash

From: Larry Rosenman <ler_at_lerctr.org>
Date: Mon, 25 May 2009 08:04:18 -0500 (CDT)
On Mon, 25 May 2009, Larry Rosenman wrote:

> after looking at the code, never mind the "don't call doadump", so we'll
> get the textdump.
>
> Thanks rwatson for the textdump stuff!
>
Here is current stats before we crash.  Does any of this look totally
out of line?


uptime:
  8:02AM  up  7:27, 1 user, load averages: 4.51, 4.72, 5.15
vmstat -m:
          Type InUse MemUse HighUse Requests  Size(s)
     pfs_nodes    20     5K       -       20  256
   pfs_vncache     2     1K       -     3138  64
      acpitask     0     0K       -        5  64
CAM dev queue     8     1K       -        8  128
          GEOM   144    38K       -     1178  16,32,64,128,256,512,1024,2048
     CAM queue    26     1K       -      132  16,32
        isadev     4     1K       -        4  128
       acpisem    17     2K       -       17  64
   ata_generic     7     7K       -      417  16,512,1024
     ad_driver     6     1K       -        6  32
    acd_driver     1     2K       -        1  2048
          cdev    12     3K       -       12  256
       ata_cam     7     7K       -       19  64,256,1024
       CAM SIM     8     2K       -        8  256
         sigio     1     1K       -        1  64
filedesc_to_leader    32     2K       -       32  64
      filedesc   171   112K       -   279497  16,32,64,128,512,1024,2048,4096
          kenv    68    11K       -       74  16,32,64,128
        kqueue     0     0K       -      724  256,2048
     proc-args    51     4K       -   214195  16,32,64,128,256
       ithread   105    19K       -      105  32,128,256
        prison     3     3K       -        3  16,256,2048
drm_ctxbitmap     1     4K       -        1  4096
        KTRACE   100    13K       -      100  128
        linker   154   188K       -      204  16,32,64,128,256,512,1024,2048,4096
         lockf    11     2K       -     5615  64,128
        ip6ndp     4     1K       -        4  64,128
          temp    37   561K       -   503816  16,32,64,128,256,512,1024,2048,4096
        devbuf 18549 41852K       -    18659  16,32,64,128,256,512,1024,2048,4096
         cache     1     1K       -        1  32
        module   227    29K       -      227  128
      mtx_pool     1     8K       -        1
           osd    10     1K       -      188  16,64,128
       subproc   536   835K       -   279698  512,4096
          proc     2    16K       -        2
       session    38     5K       -      943  128
          pgrp    45     6K       -     1463  128
          cred    64    16K       -   795986  256
       uidinfo     8     3K       -      294  64,2048
        plimit    19     5K       -     8042  256
     sysctltmp     0     0K       -    13794  16,32,64,128,2048
     sysctloid  4955   244K       -     5094  16,32,64,128
        sysctl     0     0K       -    29656  16,32,64
       callout     3  1536K       -        3
          umtx   688    86K       -      688  128
        vimage    13     1K       -       13  64
      p1003.1b     1     1K       -        1  16
          SWAP     2   549K       -        4  64
        bus-sc   101   244K       -     1997  16,32,64,128,256,512,1024,2048,4096
           bus   795    72K       -     9329  16,32,64,128,256,512,1024
         clist     4     1K       -        4  128
       devstat    10    21K       -       10  32,4096
  eventhandler    70     6K       -       70  64,128
          kobj   135   540K       -      196  4096
      drm_maps     1     1K       -        1  128
          rman   205    25K       -      610  16,32,128
          sbuf     0     0K       -     1201  16,32,64,128,256,512,1024,2048,4096
    drm_driver     4     3K       -       40  16,32,128,512,2048
     drm_sarea     1     1K       -        1  16
         stack     0     0K       -       10  256
     taskqueue    15     2K       -       15  16,32,128
        Unitno    15     1K       -    41247  32,64
           iov     0     0K       -    54811  16,64,128,256,512,1024,2048,4096
        select   217    28K       -      217  128
      ioctlops     0     0K       -    84981  16,32,64,128,256,512,1024,4096
           msg     4    30K       -        4  2048,4096
           sem     4   344K       -        4
           shm     4    26K       -      466  2048
           tty    20    20K       -       26  1024
           pts     1     1K       -        7  256
      mbuf_tag     0     0K       -     2710  32,128
         shmfd     1     8K       -        1
           pcb    43   158K       -     1607  16,32,128,1024,2048,4096
        soname     7     1K       -    29551  16,32,128
           acl     0     0K       -    19822  4096
      vfscache     1  1024K       -        1
   export_host     2     1K       -        2  256
      vfs_hash     1   512K       -        1
        vnodes     2     1K       -        2  256
   vnodemarker     0     0K       -    54735  512
         mount   787    31K       -     1938  16,32,64,128,256,512
    CAM periph     5     2K       -       14  16,32,64,128,256
           BPF     3     1K       -        3  128
   ether_multi    17     1K       -       18  16,32,64
        ifaddr    43    12K       -       43  32,64,128,256,512,4096
         ifnet     4     7K       -        4  256,2048
         clone     4    16K       -        4  4096
        arpcom     2     1K       -        2  16
       lltable    17     6K       -       18  256,512
       CAM XPT    63    21K       -      171  32,64,128,256,1024
       acpidev    57     4K       -       57  64
      routetbl    25    69K       -      275  32,64,128,256,512,1024
          igmp     3     1K       -        3  256
      pci_link    16     2K       -       16  32,64,128
      in_multi     2     1K       -        2  256
        kbdmux     7     9K       -        7  16,256,512,2048,4096
     sctp_iter     0     0K       -        3  256
      sctp_ifn     2     1K       -        2  128
      sctp_ifa     4     1K       -        4  128
      sctp_vrf     1     1K       -        1  64
     sctp_a_it     0     0K       -        3  16
     hostcache     1    28K       -        1
      syncache     1    96K       -        1
  ip6_moptions     2     1K       -        2  32,256
     in6_multi    13     2K       -       13  32,256
   in6_mfilter     1     1K       -        1  1024
           mld     3     1K       -        3  128
       NFS FHA     1     2K       -        1  2048
           NLM     0     0K       -        1  32
        crypto     1     1K       -        1  512
         xform     0     0K       -      280  16,32
           rpc    64    16K       -      194  16,32,64,128,256,512,1024
audit_evclass   170     6K       -      209  32
        newblk     1     1K       -        1  512
      inodedep     1   512K       -        1
       pagedep     1   128K       -        1
   ufs_dirhash    18     4K       -       18  16,128,512
     ufs_mount     3     7K       -        3  512,2048,4096
   ddb_capture     1  5120K       -        2
     acpi_perf     4     1K       -        4  64
       scsi_cd     0     0K       -        3  16
     vm_pgdata     2   129K       -        3  128
        acpica  1662   148K       -    47453  16,32,64,128,256,512,1024
      ppbusdev     2     1K       -        2  256
       entropy  1024    64K       -     1024  64
          UART     6     4K       -        6  16,512,1024
       io_apic     2     4K       -        2  2048
        USBdev    26    11K       -       26  64,128,512,1024,4096
           USB    23    29K       -       25  32,64,128,2048
       memdesc     1     4K       -        1  4096
           msi     2     1K       -        2  128
      nexusdev     5     1K       -        5  16
        DEVFS1   124    62K       -      138  512
        DEVFS3   302    76K       -      322  256
        DEVFS2   124     2K       -      131  16
    DEVFS_RULE    37    17K       -       37  64,512
         DEVFS    39     1K       -       40  16,128
      atkbddev     2     1K       -        2  64
        DEVFSP     1     1K       -        1  64
          ipmi     0     0K       -       11  128
   ntfs_nthash     1   512K       -        1
       solaris 3058263 1942895K       - 343220775  16,32,64,128,256,512,1024,2048,4096
    kstat_data     2     1K       -        2  64
        cpuctl     1     1K       -        1  32
         linux    64     4K       -       64  16,64
   nullfs_node     5     1K       -  3615730  32
   nullfs_hash     1     1K       -        1  128
  nullfs_mount     1     1K       -        1  16
   fdesc_mount     1     1K       -        1  16
vmstat -z:
ITEM                     SIZE     LIMIT      USED      FREE  REQUESTS  FAILURES

UMA Kegs:                 208,        0,       96,        6,       96,        0
UMA Zones:                320,        0,       96,        0,       96,        0
UMA Slabs:                568,        0,   104395,     1221,  5410178,        0
UMA RCntSlabs:            568,        0,      787,        4,      787,        0
UMA Hash:                 256,        0,        2,       13,        2,        0
16 Bucket:                152,        0,      157,       18,      157,        0
32 Bucket:                280,        0,      159,        9,      167,        2
64 Bucket:                536,        0,      205,        5,      213,        1
128 Bucket:              1048,        0,     4990,      248,     8123,    20854
VM OBJECT:                200,        0,    24683,    11930,  4998619,        0
MAP:                      224,        0,        7,       27,        7,        0
KMAP ENTRY:               112,   533544,     1360,      785, 12863094,        0
MAP ENTRY:                112,        0,     1754,     2041,  9457224,        0
DP fakepg:                112,        0,        0,        0,        0,        0
mt_zone:                 2056,        0,      268,       36,      268,        0
16:                        16,        0,   351008,      280,  9432601,        0
32:                        32,        0,     3033,    16864, 11889389,        0
64:                        64,        0,   113584,    56152, 167022544,        0
128:                      128,        0,  1189216,     3960, 29193872,        0
256:                      256,        0,    42377,   124618, 117424160,        0
512:                      512,        0,  1244737,     5540,  6407362,        0
1024:                    1024,        0,    29528,     1540,   703991,        0
2048:                    2048,        0,    27725,     1225,   460411,        0
4096:                    4096,        0,    24990,     2250,  1452188,        0
Files:                     80,        0,      210,      600,  3836592,        0
TURNSTILE:                136,        0,      689,       91,      689,        0
umtx pi:                   96,        0,        0,        0,        0,        0
PROC:                    1112,        0,      185,      187,   279348,        0
THREAD:                   912,        0,      405,      283,    21003,        0
SLEEPQUEUE:                64,        0,      689,      151,      689,        0
VMSPACE:                  384,        0,       52,      168,   279183,        0
cpuset:                    72,        0,        3,       97,        4,        0
audit_record:             984,        0,        0,        0,        0,        0
mbuf_packet:              256,        0,      262,      890, 17870314,        0
mbuf:                     256,        0,        1,     1157, 13441222,        0
mbuf_cluster:            2048,    25600,     1152,       88,     1186,        0
mbuf_jumbo_page:         4096,    12800,        0,      167,   546435,        0
mbuf_jumbo_9k:           9216,     6400,        0,        0,        0,        0
mbuf_jumbo_16k:         16384,     3200,        0,        0,        0,        0
mbuf_ext_refcnt:            4,        0,        0,        0,        0,        0
ttyinq:                   160,        0,      135,      177,      855,        0
g_bio:                    216,        0,        0,     1080,  6648699,        0
ttyoutq:                  256,        0,       72,       48,      456,        0
ata_request:              312,        0,        0,      528,  2656341,        0
ata_composite:            336,        0,        0,        0,        0,        0
taskq_ent_cache:           64,        0,      662,    16418,  1226963,        0
taskq_cache:              288,        0,       42,       23,       55,        0
cryptop:                   88,        0,        0,        0,        0,        0
cryptodesc:                72,        0,        0,        0,        0,        0
VNODE:                    472,        0,    37105,      263,  5183264,        0
VNODEPOLL:                104,        0,        0,        0,        0,        0
S VFS Cache:              108,        0,    30484,     9578,  1873995,        0
L VFS Cache:              328,        0,     1059,    19665,    77913,        0
NAMEI:                   1024,        0,        0,       96, 14057836,        0
DIRHASH:                 1024,        0,       36,       20,       36,        0
NFSMOUNT:                 608,        0,        0,        0,        0,        0
NFSNODE:                  648,        0,        0,        0,        0,        0
pipe:                     712,        0,        6,       99,   114556,        0
ksiginfo:                 112,        0,      354,      702,      354,        0
itimer:                   344,        0,        1,       21,        1,        0
KNOTE:                    120,        0,        0,      248,      756,        0
socket:                   648,    25602,       70,       98,     3118,        0
ipq:                       56,      819,        0,        0,        0,        0
udpcb:                    304,    25608,       27,       81,     1385,        0
inpcb:                    304,    25608,       26,      106,      693,        0
tcpcb:                    744,    25600,       26,       79,      693,        0
tcptw:                     88,     5124,        0,      252,      221,        0
syncache:                 136,    15372,        0,      140,      213,        0
hostcache:                136,    15372,        4,      108,       12,        0
tcpreass:                  40,     1680,        0,      336,    51642,        0
sackhole:                  32,        0,        0,      202,      232,        0
sctp_ep:                 1216,    25602,        0,        0,        0,        0
sctp_asoc:               2176,    40000,        0,        0,        0,        0
sctp_laddr:                48,    80064,        0,      216,        3,        0
sctp_raddr:               592,    80004,        0,        0,        0,        0
sctp_chunk:               144,   400010,        0,        0,        0,        0
sctp_readq:               104,   400032,        0,        0,        0,        0
sctp_stream_msg_out:       96,   400026,        0,        0,        0,        0
sctp_asconf:               40,   400008,        0,        0,        0,        0
sctp_asconf_ack:           48,   400032,        0,        0,        0,        0
ripcb:                    304,    25608,        0,       24,        1,        0
unpcb:                    240,    25600,       16,      128,     1035,        0
rtentry:                  200,        0,        9,       48,       10,        0
zio_cache:                720,        0,        0,    71570, 25615427,        0
dmu_buf_impl_t:           224,        0,  1650809,    14630,  2777078,        0
dnode_t:                  768,        0,  1138123,      322,  1169927,        0
arc_buf_hdr_t:            208,        0,   693843,     5745,  1715649,        0
arc_buf_t:                 72,        0,   538772,     8878,  1765028,        0
zil_lwb_cache:            200,        0,       11,     1471,    71316,        0
zfs_znode_cache:          376,        0,    36612,       98,  1561334,        0
selfd:                     56,        0,      291,      654,  2064138,        0
ip4flow:                   56,     4158,        0,        0,        0,        0
ip6flow:                   80,     4140,        0,        0,        0,        0
SWAPMETA:                 288,   116519,        0,        0,        0,        0
Mountpoints:              752,        0,       35,       20,       35,        0
FFS inode:                168,        0,      391,     2513,     2850,        0
FFS1 dinode:              128,        0,        0,        0,        0,        0
FFS2 dinode:              256,        0,      391,     2489,     2849,        0

-- 
Larry Rosenman                     http://www.lerctr.org/~ler
Phone: +1 512-248-2683                 E-Mail: ler_at_lerctr.org
US Mail: 430 Valona Loop, Round Rock, TX 78681-3893
Received on Mon May 25 2009 - 11:04:35 UTC

This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:39:48 UTC