search

openzfs / zfs

OpenZFS on Linux and FreeBSD
https://openzfs.github.io/openzfs-docs
Other
10.52k stars 1.74k forks source link

panic with test /opt/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective #14321

Open ikozhukhov opened 1 year ago

ikozhukhov commented 1 year ago

System information

Type Version/Name
Distribution Name DilOS
Distribution Version
Kernel Version
Architecture amd64
OpenZFS Version mix

Describe the problem you're observing

we can see degradation/panic with commit https://github.com/openzfs/zfs/commit/e8cf3a4f7662f2d1c13684ce52b73ab0d9a12266

Describe how to reproduce the problem

we have ZTS VM under vmware with nvme emulated drives.

try to run list of zfs_receive tests we can see: ZTS run /opt/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective

panic[cpu0]/thread=fffffe0010659c20: assertion failed: BP_IS_EMBEDDED(bp) || MIN(zp->zp_copies + BP_IS_GANG(bp), spa_max_replication(spa)) == BP_GET_NDVAS(bp), file: ../../common/fs/zfs/zio.c, line: 1712

fffffe0010659980 fffffffffbdcbe05 ()
fffffe0010659a00 zfs:zio_write_compress+d51 ()
fffffe0010659a40 zfs:zio_execute+116 ()
fffffe0010659b00 genunix:taskq_thread+30f ()
fffffe0010659b10 unix:thread_start+b ()

Include any warning/errors/backtraces from the system logs

Welcome to kmdb
kmdb: unable to determine terminal type: assuming `vt100'
Loaded modules: [ scsi_vhci crypto mac cpc uppc mr_sas neti sd ptm ufs unix mpt zfs krtld ipc sata apix uhci pmcs hook genunix idm ip logindmux usba xhci specfs pcplusmp nfs random mm cpu.generic arp mpt_sas emlxs sockfs ]
[0]> $C
fffffffffbca03f0 kmdb_enter+0xb()
fffffffffbca0420 debug_enter+0x53(fffffffffb973830)
fffffffffbca0510 panicsys+0x55b(fffffffffbf1fc58, fffffe00106598d8, fffffffffbca0520, 1)
fffffe00106598c0 vpanic+0x15c()
fffffe0010659930 0xfffffffffb8bd6a1()
fffffe0010659980 0xfffffffffbdcbe05()
fffffe0010659a00 zfs`zio_write_compress+0xd51(fffffe147a29e2b0)
fffffe0010659a40 zfs`zio_execute+0x116(fffffe147a29e2b0)
fffffe0010659b00 taskq_thread+0x30f(fffffe0c13b4c040)
fffffe0010659b10 thread_start+0xb()
[0]> fffffe147a29e2b0::print zio_t
{
    io_bookmark = {
        zb_objset = 0x132
        zb_object = 0
        zb_level = 0xffffffffffffffff
        zb_blkid = 0
    }
    io_prop = {
        zp_checksum = 7 (ZIO_CHECKSUM_FLETCHER_4)
        zp_compress = 4 (ZIO_COMPRESS_EMPTY)
        zp_complevel = 0
        zp_type = 0t11 (DMU_OT_OBJSET)
        zp_level = 0
        zp_copies = 0x2
        zp_dedup = 0 (0)
        zp_dedup_verify = 0 (0)
        zp_nopwrite = 0 (0)
        zp_encrypt = 0x1 (B_TRUE)
        zp_byteorder = 0x1 (B_TRUE)
        zp_salt = [ 0, 0, 0, 0, 0, 0, 0, 0 ]
        zp_iv = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]
        zp_mac = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]
        zp_zpl_smallblk = 0
    }
    io_type = 2 (ZIO_TYPE_WRITE)
    io_child_type = 3 (ZIO_CHILD_LOGICAL)
    io_trim_flags = 0 (0)
    io_cmd = 0
    io_priority = 3 (ZIO_PRIORITY_ASYNC_WRITE)
    io_reexecute = 0
    io_state = [ 0, 0 ]
    io_txg = 0xa0
    io_spa = 0xfffffe22e4870000
    io_bp = 0xfffffe0bf2556900
    io_bp_override = 0
    io_bp_copy = {
        blk_dva = [
            {
                dva_word = [ 0x9, 0x8000000000300119 ]
            },
            {
                dva_word = [ 0x9, 0x8000000000400119 ]
            },
            {
                dva_word = [ 0, 0 ]
            },
        ]
        blk_prop = 0xa00b070200070007
        blk_pad = [ 0, 0 ]
        blk_phys_birth = 0
        blk_birth = 0xa0
        blk_fill = 0x8
        blk_cksum = {
            zc_word = [ 0x939053322, 0x213a1314ce55, 0x3c4bd6a077fb2b, 0x4977bcb1ce3640bc ]
        }
    }
    io_parent_list = {
        list_size = 0x30
        list_offset = 0x10
        list_head = {
            list_next = 0xfffffe12de6c2388
            list_prev = 0xfffffe12de6c2388
        }
    }
    io_child_list = {
        list_size = 0x30
        list_offset = 0x20
        list_head = {
            list_next = 0xfffffe139d2bc7f8
            list_prev = 0xfffffe142a19eda8
        }
    }
    io_logical = 0xfffffe147a29e2b0
    io_transform_stack = 0
    io_ready = zfs`arc_write_ready
    io_children_ready = 0
    io_physdone = zfs`arc_write_physdone
    io_done = zfs`arc_write_done
    io_private = 0xfffffe0bf0a23d28
    io_prev_space_delta = 0
    io_bp_orig = {
        blk_dva = [
            {
                dva_word = [ 0x9, 0x8000000000300119 ]
            },
            {
                dva_word = [ 0x9, 0x8000000000400119 ]
            },
            {
                dva_word = [ 0, 0 ]
            },
        ]
        blk_prop = 0xa00b070200070007
        blk_pad = [ 0, 0 ]
        blk_phys_birth = 0
        blk_birth = 0xa0
        blk_fill = 0x8
        blk_cksum = {
            zc_word = [ 0x939053322, 0x213a1314ce55, 0x3c4bd6a077fb2b, 0x4977bcb1ce3640bc ]
        }
    }
    io_lsize = 0x1000
    io_abd = 0xfffffe13ca6b9100
    io_orig_abd = 0xfffffe13ca6b9100
    io_size = 0x1000
    io_orig_size = 0x1000
    io_vd = 0
    io_vsd = 0
    io_vsd_ops = 0
    io_metaslab_class = 0
    io_offset = 0
    io_timestamp = 0
    io_queued_timestamp = 0x39be4021041
    io_target_timestamp = 0
    io_dispatched = 0
    io_delta = 0
    io_delay = 0
    io_queue_node = {
        avl_child = [ 0, 0 ]
        avl_pcb = 0
    }
    io_offset_node = {
        avl_child = [ 0, 0 ]
        avl_pcb = 0
    }
    io_alloc_node = {
        avl_child = [ 0, 0 ]
        avl_pcb = 0
    }
    io_alloc_list = {
        zal_list = {
            list_size = 0x48
            list_offset = 0
            list_head = {
                list_next = 0xfffffe147a29e5b0
                list_prev = 0xfffffe147a29e5b0
            }
        }
        zal_size = 0
    }
    io_flags = 0
    io_stage = 0x20 (ZIO_STAGE_WRITE_COMPRESS)
    io_pipeline = 0x179e0f4 (ZIO_STAGE_{WRITE_BP_INIT|ISSUE_ASYNC|WRITE_COMPRESS|ENCRYPT|CHECKSUM_GENERATE|GANG_ASSEMBLE|GANG_ISSUE|DVA_THROTTLE|DVA_ALLOCATE|READY|VDEV_IO_START|VDEV_IO_DONE|VDEV_IO_ASSESS|DONE})
    io_orig_flags = 0
    io_orig_stage = 0x1 (ZIO_STAGE_OPEN)
    io_orig_pipeline = 0x179e0f4 (ZIO_STAGE_{WRITE_BP_INIT|ISSUE_ASYNC|WRITE_COMPRESS|ENCRYPT|CHECKSUM_GENERATE|GANG_ASSEMBLE|GANG_ISSUE|DVA_THROTTLE|DVA_ALLOCATE|READY|VDEV_IO_START|VDEV_IO_DONE|VDEV_IO_ASSESS|DONE})
    io_pipeline_trace = 0x35 (ZIO_STAGE_{OPEN|WRITE_BP_INIT|ISSUE_ASYNC|WRITE_COMPRESS})
    io_error = 0
    io_child_error = [ 0, 0, 0, 0 ]
    io_children = [
        [ 0, 0 ]
        [ 0, 0 ]
        [ 0, 0 ]
        [ 0, 0x3 ]
    ]
    io_child_count = 0x3
    io_phys_children = 0
    io_parent_count = 0x1
    io_stall = 0
    io_gang_leader = 0
    io_gang_tree = 0
    io_executor = 0xfffffe0010659c20
    io_waiter = 0
    io_lock = {
        _opaque = [ 0 ]
    }
    io_cv = {
        _opaque = 0
    }
    io_allocator = 0
    io_cksum_report = 0
    io_ena = 0
    io_zoneid = 0
    io_tqent = {
        tqent_next = 0
        tqent_prev = 0
        tqent_func = zfs`zio_execute
        tqent_arg = 0xfffffe147a29e2b0
        tqent_id = 0
        tqent_un = {
            tqent_bucket = 1
            tqent_flags = 0x1
        }
        tqent_thread = 0
        tqent_cv = {
            _opaque = 0
        }
        tqent_cv_done = {
            _opaque = 0
        }
    }
}
[0]> ::status
debugging live kernel (64-bit) on zfs-t3
operating system: 5.11 2.0.3.8-20221224-1-0+int2-ozfs (i86pc)
image uuid: f5ec46f7-fcc9-4441-ad59-8a8eb637fe99
DTrace state: inactive
stopped on: debugger entry trap
ikozhukhov commented 1 year ago

@behlendorf @ahrens

ikozhukhov commented 1 year ago

we can see panic only with NVME emulated drives under vmware guests. we have no panic with SAS or SATA emulated drives