Re: [ZFS][PANIC] Solaris Assert/zio.c:2548

On 21/07/14 01:46, Steven Hartland wrote:
> ----- Original Message ----- From: "Larry Rosenman" <ler_at_lerctr.org>
> To: "Steven Hartland" <killing_at_multiplay.co.uk>
> Cc: <freebsd-fs_at_freebsd.org>; <freebsd-current_at_freebsd.org>
> Sent: Monday, July 21, 2014 12:22 AM
> Subject: Re: [ZFS][PANIC] Solaris Assert/zio.c:2548
> 
> 
>> On 2014-07-20 18:21, Steven Hartland wrote:
>>> Can you try reverting r265321 and see if you still see the
>>> same crash?
>>>
>>>    Regards
>>>    Steve
>> I'll do the revert, but it's been a ONE TIME hit.
>>
>> There was a followup to mine with a reproducible poudriere crash like
>> mine.
> 
> If you don't have a reproducable senario I'd hold off.
> 
> Florian, is yours reproducable and can you send me
> a pretty print of the crashing zio?
> 

My backtrace looks a little different.

panic: solaris assert: !(zio->io_flags & ZIO_FLAG_DELEGATED), file:
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c,
line: 2874
cpuid = 3
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame
0xfffffe00002e97f0
kdb_backtrace() at kdb_backtrace+0x39/frame 0xfffffe00002e98a0
vpanic() at vpanic+0x126/frame 0xfffffe00002e98e0
panic() at panic+0x43/frame 0xfffffe00002e9940
assfail() at assfail+0x1d/frame 0xfffffe00002e9950
zio_vdev_io_assess() at zio_vdev_io_assess+0x2e8/frame 0xfffffe00002e9980
zio_execute() at zio_execute+0x1e9/frame 0xfffffe00002e99e0
taskqueue_run_locked() at taskqueue_run_locked+0xf0/frame 0xfffffe00002e9a40
taskqueue_thread_loop() at taskqueue_thread_loop+0x9b/frame
0xfffffe00002e9a70
fork_exit() at fork_exit+0x84/frame 0xfffffe00002e9ab0
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00002e9ab0
--- trap 0, rip = 0, rsp = 0xfffffe00002e9b70, rbp = 0 ---
KDB: enter: panic
(kgdb) where
#0  doadump (textdump=-2125462752) at pcpu.h:219
#1  0xffffffff80347655 in db_fncall (dummy1=<value optimized out>,
    dummy2=<value optimized out>, dummy3=<value optimized out>,
    dummy4=<value optimized out>) at /usr/src/sys/ddb/db_command.c:578
#2  0xffffffff8034733d in db_command (cmd_table=0x0)
    at /usr/src/sys/ddb/db_command.c:449
#3  0xffffffff803470b4 in db_command_loop ()
    at /usr/src/sys/ddb/db_command.c:502
#4  0xffffffff80349a90 in db_trap (type=<value optimized out>, code=0)
    at /usr/src/sys/ddb/db_main.c:231
#5  0xffffffff80944159 in kdb_trap (type=3, code=0, tf=<value optimized
out>)
    at /usr/src/sys/kern/subr_kdb.c:654
#6  0xffffffff80d1e532 in trap (frame=0xfffffe00002e97d0)
    at /usr/src/sys/amd64/amd64/trap.c:542
#7  0xffffffff80d01202 in calltrap ()
    at /usr/src/sys/amd64/amd64/exception.S:231
#8  0xffffffff809438be in kdb_enter (why=0xffffffff80f9ce38 "panic",
    msg=<value optimized out>) at cpufunc.h:63
#9  0xffffffff8090bb66 in vpanic (fmt=<value optimized out>,
    ap=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:737
#10 0xffffffff8090bbd3 in panic (fmt=0xffffffff815a59a0 "\004")
    at /usr/src/sys/kern/kern_shutdown.c:673
#11 0xffffffff81fb821d in assfail (a=<value optimized out>,
---Type <return> to continue, or q <return> to quit---
    f=<value optimized out>, l=<value optimized out>)
    at /usr/src/sys/cddl/compat/opensolaris/kern/opensolaris_cmn_err.c:81
#12 0xffffffff81eca848 in zio_vdev_io_assess (ziop=<value optimized out>)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:2874
#13 0xffffffff81ec58b9 in zio_execute (zio=0xfffff801a8abc398)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:1416
#14 0xffffffff80954150 in taskqueue_run_locked (queue=0xfffff80009249b00)
    at /usr/src/sys/kern/subr_taskqueue.c:356
#15 0xffffffff80954c1b in taskqueue_thread_loop (arg=<value optimized out>)
    at /usr/src/sys/kern/subr_taskqueue.c:623
#16 0xffffffff808d9834 in fork_exit (
    callout=0xffffffff80954b80 <taskqueue_thread_loop>,
    arg=0xfffff80003dfeed0, frame=0xfffffe00002e9ac0)
    at /usr/src/sys/kern/kern_fork.c:977
#17 0xffffffff80d0173e in fork_trampoline ()
    at /usr/src/sys/amd64/amd64/exception.S:605
#18 0x0000000000000000 in ?? ()
(kgdb) frame 12
#12 0xffffffff81eca848 in zio_vdev_io_assess (ziop=<value optimized out>)
    at
/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:2874
2874                    ASSERT(!(zio->io_flags & ZIO_FLAG_DELEGATED));
(kgdb) print zio
$3 = (zio_t *) 0xfffff801a8abc398
(kgdb) print *zio
$4 = {io_bookmark = {zb_objset = 4339, zb_object = 327827, zb_level = 0,
    zb_blkid = 0}, io_prop = {zp_checksum = ZIO_CHECKSUM_INHERIT,
    zp_compress = ZIO_COMPRESS_INHERIT, zp_type = DMU_OT_NONE,
    zp_level = 0 '\0', zp_copies = 0 '\0', zp_dedup = 0, zp_dedup_verify
= 0,
    zp_nopwrite = 0}, io_type = ZIO_TYPE_WRITE,
  io_child_type = ZIO_CHILD_VDEV, io_cmd = 0,
  io_priority = ZIO_PRIORITY_ASYNC_WRITE, io_reexecute = 0 '\0',
  io_state = "\001", io_txg = 1312558, io_spa = 0xfffffe00022e6000,
  io_bp = 0xfffffe000a94a640, io_bp_override = 0x0, io_bp_copy =
{blk_dva = {{
        dva_word = {1, 58754170}}, {dva_word = {1, 69614673}}, {dva_word
= {0,
          0}}}, blk_prop = 9229009297394892802, blk_pad = {0, 0},
    blk_phys_birth = 0, blk_birth = 1312558, blk_fill = 1, blk_cksum = {
      zc_word = {72684358009, 6982033287555, 350329209490535,
        12175142665158025}}}, io_parent_list = {list_size = 48,
    list_offset = 16, list_head = {list_next = 0xfffff800092e7520,
      list_prev = 0xfffff800092e7520}}, io_child_list = {list_size = 48,
    list_offset = 32, list_head = {list_next = 0xfffff801a8abc4b8,
      list_prev = 0xfffff801a8abc4b8}}, io_walk_link = 0x0,
  io_logical = 0xfffff801a8cb7730, io_transform_stack = 0x0, io_ready = 0,
  io_physdone = 0xffffffff81e34ee0 <arc_write_physdone>,
  io_done = 0xffffffff81ea7ea0 <vdev_mirror_child_done>,
  io_private = 0xfffff80027266d18, io_prev_space_delta = 0, io_bp_orig = {
    blk_dva = {{dva_word = {1, 58754170}}, {dva_word = {1, 69614673}}, {
---Type <return> to continue, or q <return> to quit---
        dva_word = {0, 0}}}, blk_prop = 9229009297394892802, blk_pad =
{0, 0},
    blk_phys_birth = 0, blk_birth = 1312558, blk_fill = 1, blk_cksum = {
      zc_word = {72684358009, 6982033287555, 350329209490535,
        12175142665158025}}}, io_data = 0xfffff8017b0b3000,
  io_orig_data = 0xfffff8017b0b3000, io_size = 512, io_orig_size = 512,
  io_vd = 0xfffff8000935f800, io_vsd = 0x0, io_vsd_ops = 0x0,
  io_offset = 30086329344, io_timestamp = 76492632312, io_queue_node = {
    avl_child = {0x0, 0x0}, avl_pcb = 18446735284737973505},
  io_flags = 269224064, io_stage = ZIO_STAGE_VDEV_IO_ASSESS,
  io_pipeline = 3014656, io_orig_flags = 524416,
  io_orig_stage = ZIO_STAGE_READY, io_orig_pipeline = 3014656, io_error = 0,
  io_child_error = {0, 0, 0, 0}, io_children = {{0, 0}, {0, 0}, {0, 0}, {0,
      0}}, io_child_count = 0, io_phys_children = 0, io_parent_count = 1,
  io_stall = 0x0, io_gang_leader = 0x0, io_gang_tree = 0x0,
  io_executor = 0xfffff80009265490, io_waiter = 0x0, io_lock =
{lock_object = {
      lo_name = 0xffffffff81f763a5 "zio->io_lock", lo_flags = 40960000,
      lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, io_cv = {
    cv_description = 0xffffffff81f763b3 "zio->io_cv", cv_waiters = 0},
  io_cksum_report = 0x0, io_ena = 0, io_tqent = {tqent_task = {ta_link = {
        stqe_next = 0x0}, ta_pending = 0, ta_priority = 0,
      ta_func = 0xffffffff81e1dd60 <taskq_run_ent>,
      ta_context = 0xfffff801a8abc6d8},
    tqent_func = 0xffffffff81ec56d0 <zio_execute>,
---Type <return> to continue, or q <return> to quit---
    tqent_arg = 0xfffff801a8abc398}, io_trim_node = {avl_child = {0x0, 0x0},
    avl_pcb = 0}, io_trim_link = {list_next = 0x0, list_prev = 0x0}}

I'm not an expert with kgdb. If you need anything else let me know what
you need.

Florian