search

openzfs / zfs

OpenZFS on Linux and FreeBSD
https://openzfs.github.io/openzfs-docs
Other
10.56k stars 1.74k forks source link

ztest: zap_update() ASSERT #5501

Closed jsalinasintel closed 5 years ago

jsalinasintel commented 7 years ago

Version Information

Distribution Name Centos 7.2
Distribution Version 7.2 Linux Kernel 3.10.0-327.36.3.el7.x86_64 Architecture x86_64 ZFS Version master Dec 1st RPM/modinfo say: 0.7.0-rc2 SPL Version master Dec 1st RPM/modinfo say: 0.7.0-rc2
Git Build Data Revision: 5f20c145df3ca45f86cd1892137edb117d4c8a82 refs/remotes/origin/master Built Branches refs/remotes/origin/master: Build #183 of Revision 5f20c145df3ca45f86cd1892137edb117d4c8a82 (refs/remotes/origin/master)

Problem

Running zloop we hit an assert: zap_update(os, object, name, wsize, wc, data, tx) == 0 ASSERT at ztest.c:4752:ztest_zap_parallel()/usr/sbin/ztest[0x40a7cf]

Reproduce =================================================================
/sbin/ztest -VVVVV -m 1 -r 9 -R 3 -v 3 -a 12 -T 68 -P 20 -s 128m -f /var/tmp

Log detail

ztest.out 

 starting main threads...
  0.00 sec in ztest_dmu_read_write
  0.01 sec in ztest_fault_inject
  0.04 sec in ztest_fzap
  0.00 sec in ztest_split_pool
  0.00 sec in ztest_split_pool
  0.64 sec in ztest_spa_create_destroy
  0.62 sec in ztest_spa_create_destroy
  0.00 sec in ztest_split_pool
  0.00 sec in ztest_split_pool
  0.00 sec in ztest_zil_commit
  0.00 sec in ztest_zil_remount
  0.00 sec in ztest_dmu_commit_callbacks
  0.01 sec in ztest_dmu_commit_callbacks
  0.01 sec in ztest_dmu_read_write
  0.00 sec in ztest_split_pool
  0.01 sec in ztest_dmu_read_write
  0.01 sec in ztest_dmu_read_write
  0.02 sec in ztest_fault_inject
zap_update(os, object, name, wsize, wc, data, tx) == 0
ASSERT at ztest.c:4752:ztest_zap_parallel()/usr/sbin/ztest[0x40a7cf]
/lib64/libpthread.so.0(+0xf100)[0x7f85aa917100]
/lib64/libc.so.6(gsignal+0x37)[0x7f85aa57b5f7]
/lib64/libc.so.6(abort+0x148)[0x7f85aa57cce8]
/usr/sbin/ztest[0x4089dd]
/usr/sbin/ztest[0x412272]
/usr/sbin/ztest[0x40cda3]
/lib64/libzpool.so.2(zk_thread_helper+0x1ec)[0x7f85abb0b7ac]
/lib64/libpthread.so.0(+0x7dc5)[0x7f85aa90fdc5]
/lib64/libc.so.6(clone+0x6d)[0x7f85aa63cced]
child died with signal 6

From status 

[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/usr/sbin/ztest'.
Program terminated with signal 6, Aborted.
#0  0x00007f85aa57b5f7 in raise () from /lib64/libc.so.6
*
* Backtrace 
*
#0  0x00007f85aa57b5f7 in raise () from /lib64/libc.so.6
#1  0x00007f85aa57cce8 in abort () from /lib64/libc.so.6
#2  0x00000000004089dd in libspl_assert ()
#3  0x0000000000412272 in ztest_zap_parallel ()
#4  0x000000000040cda3 in ztest_thread ()
#5  0x00007f85abb0b7ac in zk_thread_helper () from /lib64/libzpool.so.2
#6  0x00007f85aa90fdc5 in start_thread () from /lib64/libpthread.so.0
#7  0x00007f85aa63cced in clone () from /lib64/libc.so.6

Thread 742 (Thread 0x7f8591660700 (LWP 71882)):
#0  0x00007f85aa91684d in fsync () from /lib64/libpthread.so.0
#1  0x00007f85abb8a06c in vdev_file_io_start () from /lib64/libzpool.so.2
#2  0x00007f85abc02f27 in zio_vdev_io_start () from /lib64/libzpool.so.2
#3  0x00007f85abc01491 in zio_nowait () from /lib64/libzpool.so.2
#4  0x00007f85abc015a4 in zio_ioctl () from /lib64/libzpool.so.2
#5  0x00007f85abc01662 in zio_flush () from /lib64/libzpool.so.2
#6  0x00007f85abb8c3f3 in vdev_config_sync () from /lib64/libzpool.so.2
#7  0x00007f85abb72fea in spa_sync () from /lib64/libzpool.so.2
#8  0x00007f85abb820a9 in txg_sync_thread () from /lib64/libzpool.so.2
#9  0x00007f85abb0b7ac in zk_thread_helper () from /lib64/libzpool.so.2
#10 0x00007f85aa90fdc5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007f85aa63cced in clone () from /lib64/libc.so.6

Thread 287 (Thread 0x7f85ac6687c0 (LWP 70464)):
#0  0x00007f85aa910ef7 in pthread_join () from /lib64/libpthread.so.0
#1  0x00007f85abb0c2f1 in zk_thread_join () from /lib64/libzpool.so.2
#2  0x0000000000407805 in ztest_run ()
#3  0x00000000004087e4 in main ()

Thread 143 (Thread 0x7f83189a9700 (LWP 73823)):
#0  0x00007f85ac055924 in atomic_add_ptr () from /lib64/libuutil.so.1
#1  0x00007f85abb14fef in abd_free () from /lib64/libzpool.so.2
#2  0x00007f85abb1a3c5 in arc_hdr_free_pabd () from /lib64/libzpool.so.2
#3  0x00007f85abb1a7a3 in arc_hdr_destroy () from /lib64/libzpool.so.2
#4  0x00007f85abb1cb1a in arc_buf_destroy () from /lib64/libzpool.so.2
#5  0x00007f85abb24ba3 in dbuf_destroy () from /lib64/libzpool.so.2
#6  0x00007f85abb248a0 in dbuf_rele_and_unlock () from /lib64/libzpool.so.2
#7  0x00007f85abb2b9f8 in dmu_buf_rele_array () from /lib64/libzpool.so.2
#8  0x00007f85abb2c557 in dmu_read () from /lib64/libzpool.so.2
#9  0x0000000000410e98 in ztest_dmu_read_write ()
#10 0x000000000040cda3 in ztest_thread ()
#11 0x00007f85abb0b7ac in zk_thread_helper () from /lib64/libzpool.so.2
#12 0x00007f85aa90fdc5 in start_thread () from /lib64/libpthread.so.0
#13 0x00007f85aa63cced in clone () from /lib64/libc.so.6 

Thread 62 (Thread 0x7f85910ca700 (LWP 71887)):
#0  0x00007f85aa91303e in pthread_rwlock_wrlock () from /lib64/libpthread.so.0
#1  0x0000000000412769 in ztest_ddt_repair ()
#2  0x000000000040cda3 in ztest_thread ()
#3  0x00007f85abb0b7ac in zk_thread_helper () from /lib64/libzpool.so.2
#4  0x00007f85aa90fdc5 in start_thread () from /lib64/libpthread.so.0
#5  0x00007f85aa63cced in clone () from /lib64/libc.so.6

Thread 46 (Thread 0x7f85912d2700 (LWP 71889)):
#0  0x00007f85aa91622d in read () from /lib64/libpthread.so.0
#1  0x000000000040af50 in ztest_random ()
#2  0x000000000040bc6a in ztest_fletcher_incr ()
#3  0x000000000040cda3 in ztest_thread ()
#4  0x00007f85abb0b7ac in zk_thread_helper () from /lib64/libzpool.so.2
#5  0x00007f85aa90fdc5 in start_thread () from /lib64/libpthread.so.0
#6  0x00007f85aa63cced in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7f831b924700 (LWP 73820)):
#0  0x00007f85aa57b5f7 in raise () from /lib64/libc.so.6
#1  0x00007f85aa57cce8 in abort () from /lib64/libc.so.6
#2  0x00000000004089dd in libspl_assert ()
#3  0x0000000000412272 in ztest_zap_parallel ()
#4  0x000000000040cda3 in ztest_thread ()
#5  0x00007f85abb0b7ac in zk_thread_helper () from /lib64/libzpool.so.2
#6  0x00007f85aa90fdc5 in start_thread () from /lib64/libpthread.so.0
#7  0x00007f85aa63cced in clone () from /lib64/libc.so.6

zdb

Dataset mos [META], ID 0, cr_txg 4, 562K, 150 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    2    16K    16K  71.5K     512  80.0K   93.75  DMU dnode
         1    1    16K    16K  24.0K     512    32K  100.00  object directory
         2    1    16K    512      0     512    512    0.00  DSL directory
         3    1    16K    512      0     512    512  100.00  DSL props
         4    1    16K    512      0     512    512  100.00  DSL directory child map
         5    1    16K    512      0     512    512  100.00  zap
         6    1    16K    512      0     512    512  100.00  DSL dataset snap map
         7    1    16K    512      0     512    512  100.00  DSL deadlist map
         8    1    16K   128K      0     512   128K    0.00  bpobj
         9    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        10    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        11    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        12    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        13    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        14    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        15    1    16K    512      0     512    512    0.00  DSL directory
        16    1    16K    512      0     512    512  100.00  DSL props
        17    1    16K    512      0     512    512  100.00  DSL directory child map
        18    1    16K    512      0     512    512  100.00  zap
        19    1    16K    512      0     512    512  100.00  DSL dataset snap map
        20    1    16K    512      0     512    512  100.00  DSL deadlist map
        21    2    16K     4K  36.0K     512     8K  100.00  SPA space map
        22    2    16K     4K  36.0K     512     8K  100.00  SPA space map
        23    2    16K     4K  24.0K     512     4K  100.00  SPA space map
        24    1    16K    512      0     512    512    0.00  DSL directory
        25    1    16K    512      0     512    512  100.00  DSL props
        26    1    16K    512      0     512    512  100.00  DSL directory child map
        27    1    16K    512      0     512    512  100.00  zap
        28    1    16K    512      0     512    512  100.00  DSL dataset snap map
        29    1    16K    512      0     512    512  100.00  DSL deadlist map
        30    1    16K    512      0     512    512    0.00  DSL directory
        31    1    16K    512  12.0K     512    512  100.00  DSL props
        32    1    16K    512      0     512    512    0.00  DSL directory
        33    1    16K    512      0     512    512  100.00  DSL props
        34    1    16K  1.50K  12.0K     512  1.50K  100.00  DSL directory child map
        35    1    16K    512      0     512    512    0.00  DSL directory
        36    1    16K    512      0     512    512  100.00  DSL props
        37    1    16K    512      0     512    512  100.00  DSL directory child map
        38    1    16K    512      0     512    512    0.00  DSL directory
        39    1    16K    512      0     512    512  100.00  DSL props
        40    1    16K    512      0     512    512  100.00  DSL directory child map
        41    1    16K   128K      0     512   128K    0.00  bpobj
        42    1    16K    512      0     512    512    0.00  DSL directory
        43    1    16K    512      0     512    512  100.00  DSL props
        44    1    16K    512      0     512    512  100.00  DSL directory child map
        45    1    16K    512      0     512    512    0.00  DSL dataset
        46    1    16K    512      0     512    512  100.00  DSL dataset snap map
        47    1    16K    512      0     512    512  100.00  DSL deadlist map
        48    1    16K    512      0     512    512    0.00  DSL dataset
        49    1    16K    512      0     512    512  100.00  DSL deadlist map
        50    1    16K   128K      0     512   128K    0.00  bpobj
        51    1    16K    512      0     512    512    0.00  DSL dataset
        52    1    16K    512      0     512    512  100.00  DSL dataset snap map
        53    1    16K    512      0     512    512  100.00  DSL deadlist map
        54    1    16K   128K      0     512   128K    0.00  bpobj
        55    1    16K     1K  12.0K     512     1K  100.00  DSL dataset next clones
        56    1    16K     1K  12.0K     512     1K  100.00  DSL dir clones
        57    1    16K    16K  12.0K     512    16K  100.00  packed nvlist
        58    1    16K     1K  12.0K     512     1K  100.00  zap
        59    1    16K     1K  12.0K     512     1K  100.00  zap
        60    1    16K    16K  36.0K     512    32K  100.00  zap
        61    1    16K    16K  12.0K     512    16K  100.00  bpobj (Z=uncompressed)
        62    1    16K   128K  12.0K     512   128K  100.00  SPA history
        63    1    16K    512      0     512    512  100.00  Pool properties
        64    1    16K     1K  12.0K     512     1K  100.00  zap
        65    1    16K     1K  12.0K     512     1K  100.00  zap
        66    1    16K    512      0     512    512  100.00  zap
        67    1    16K    512      0     512    512  100.00  zap
        68    1    16K    512      0     512    512  100.00  zap
        69    1    16K    512      0     512    512  100.00  zap
        70    1    16K    512      0     512    512  100.00  object array
        71    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        72    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        73    1    16K     4K  12.0K     512     4K  100.00  SPA space map
        74    1    16K    512      0     512    512  100.00  DSL directory child map
        75    1    16K    512      0     512    512  100.00  zap
        76    1    16K    512      0     512    512  100.00  DSL dataset snap map
        77    1    16K    512      0     512    512  100.00  DSL deadlist map
        78    1    16K    512      0     512    512    0.00  DSL directory
        79    1    16K    512      0     512    512  100.00  DSL props
        80    1    16K    512      0     512    512  100.00  DSL directory child map
        81    1    16K    512      0     512    512  100.00  zap
        82    1    16K    512      0     512    512  100.00  DSL dataset snap map
        83    1    16K    512      0     512    512  100.00  DSL deadlist map
        84    1    16K    512      0     512    512    0.00  DSL directory
        85    1    16K    512      0     512    512  100.00  DSL props
        86    1    16K    512      0     512    512  100.00  DSL directory child map
        87    1    16K    512      0     512    512  100.00  zap
        88    1    16K    512      0     512    512  100.00  DSL dataset snap map
        89    1    16K    512      0     512    512  100.00  DSL deadlist map
        90    1    16K    512      0     512    512    0.00  DSL directory
        91    1    16K    512      0     512    512  100.00  DSL props
        92    1    16K    512      0     512    512  100.00  DSL directory child map
        93    1    16K    512      0     512    512  100.00  zap
        94    1    16K    512      0     512    512  100.00  DSL dataset snap map
        95    1    16K    512      0     512    512  100.00  DSL deadlist map
        96    1    16K    512      0     512    512  100.00  zap
        97    1    16K    512      0     512    512  100.00  DSL deadlist map
        98    1    16K    512      0     512    512    0.00  DSL directory
        99    1    16K    512      0     512    512  100.00  DSL props
       100    1    16K    512      0     512    512  100.00  DSL directory child map
       101    1    16K    512      0     512    512  100.00  zap
       102    1    16K    512      0     512    512  100.00  DSL dataset snap map
       103    1    16K    512      0     512    512  100.00  DSL deadlist map
       104    1    16K    512      0     512    512    0.00  DSL directory
       105    1    16K    512      0     512    512  100.00  DSL props
       106    1    16K    512      0     512    512  100.00  DSL directory child map
       107    1    16K    512      0     512    512  100.00  zap
       108    1    16K    512      0     512    512  100.00  DSL dataset snap map
       109    1    16K    512      0     512    512  100.00  DSL deadlist map
       110    1    16K    16K  24.0K     512    32K  100.00  DDT statistics
       111    1     4K     4K  12.0K     512     8K  100.00  DDT ZAP algorithm
       112    1    16K    512      0     512    512    0.00  DSL directory
       113    1    16K    512      0     512    512  100.00  DSL props
       114    1    16K    512      0     512    512  100.00  DSL directory child map
       115    1    16K    512      0     512    512  100.00  zap
       116    1    16K    512      0     512    512  100.00  DSL dataset snap map
       117    1    16K    512      0     512    512  100.00  DSL deadlist map
       118    1    16K    512      0     512    512    0.00  DSL directory
       119    1    16K    512      0     512    512  100.00  DSL props
       120    1    16K    512      0     512    512  100.00  DSL directory child map
       121    1    16K    512      0     512    512  100.00  zap
       122    1    16K    512      0     512    512  100.00  DSL dataset snap map
       123    1    16K    512      0     512    512  100.00  DSL deadlist map
       124    1    16K    512      0     512    512  100.00  zap
       125    1    16K    512      0     512    512  100.00  zap
       126    1    16K    512      0     512    512  100.00  zap
       127    1    16K    512      0     512    512  100.00  zap
       128    1    16K    512      0     512    512  100.00  object array
       129    1    16K     4K  12.0K     512     4K  100.00  SPA space map
       130    1    16K     4K  12.0K     512     4K  100.00  SPA space map
       131    1    16K     4K  12.0K     512     4K  100.00  SPA space map
       132    1    16K    512      0     512    512  100.00  zap
       133    1    16K    512      0     512    512  100.00  zap
       134    1    16K    512      0     512    512  100.00  zap
       135    1    16K    512      0     512    512  100.00  zap
       136    1    16K    512      0     512    512    0.00  DSL directory
       137    1    16K    512      0     512    512  100.00  DSL props
       138    1    16K    512      0     512    512  100.00  DSL directory child map
       139    1    16K    512      0     512    512    0.00  DSL dataset
       140    1    16K    512      0     512    512  100.00  DSL dataset snap map
       141    1    16K    512      0     512    512  100.00  DSL deadlist map
       142    1    16K    512      0     512    512  100.00  object array
       143    1    16K     4K      0     512     4K  100.00  SPA space map
       144    1    16K     4K  12.0K     512     4K  100.00  SPA space map
       145    1    16K    512      0     512    512    0.00  DSL directory
       146    1    16K    512      0     512    512  100.00  DSL props
       147    1    16K    512      0     512    512  100.00  DSL directory child map
       148    1    16K    512      0     512    512  100.00  zap
       149    1    16K    512      0     512    512  100.00  DSL dataset snap map
       150    1    16K    512      0     512    512  100.00  DSL deadlist map

Dataset ztest/temp_12 [OTHER], ID 115, cr_txg 85, 566K, 5 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  56.0K     512    16K   15.62  DMU dnode
         1    1    16K    512     8K     512    512  100.00  other ZAP
         2    4     8K     4K  87.5K      1K   928M    0.00  other uint64[]
         4    3    16K    64K   223K      1K   960M    0.02  other uint64[]
         6    4     4K   128K   152K   1.50K   304M    0.04  other uint64[]
         9    4     8K     8K    32K     512   400M    0.00  other uint64[]

Dataset ztest/temp_7 [OTHER], ID 121, cr_txg 87, 195K, 5 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  56.0K     512    16K   15.62  DMU dnode
         1    1    16K    512     8K     512    512  100.00  other ZAP
         2    5     8K    512    32K     512   400M    0.00  other uint64[]
         3    3    16K     8K  24.0K      2K  32.0M    0.02  other uint64[]
         7    4    16K    16K  28.0K     512   976M    0.00  other uint64[]
         8    4    16K    16K  40.0K     512   432M    0.00  other uint64[]

Dataset ztest/ds_2 [OTHER], ID 27, cr_txg 65, 1000K, 13 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K   108K     512  48.0K   13.54  DMU dnode
         1    1    16K     1K  12.0K     512     1K  100.00  other ZAP
         2    7     8K     8K   271K     512   343T    0.00  other uint64[]
         3    6     8K    32K   219K     512  1.01T    0.00  other uint64[]
         4    1    16K  1.50K     8K     512  1.50K  100.00  other ZAP
         5    2    16K    16K  24.0K     512    32K  100.00  other ZAP
         6    5     8K     1K  91.5K     512   784M    0.00  other uint64[]
         7    6     4K    512   104K      2K  1008M    0.00  other uint64[]
        11    5     4K     2K  36.0K     512   896M    0.00  other uint64[]
        12    4    16K     4K      0      1K   544M    0.00  other uint64[]
        72    1    16K    512      0   9.00K    512  100.00  other ZAP
        90    8     8K    64K    64K      1K  4.32P    0.00  other uint64[]
        92    7     4K    16K  52.0K     512  1.62T    0.00  other uint64[]
        93    6    16K     2K      0      1K  16.0T    0.00  other uint64[]

Dataset ztest/ds_5 [OTHER], ID 87, cr_txg 75, 566K, 6 objects

    ZIL header: claim_txg 0, claim_blk_seq 0, claim_lr_seq 0 replay_seq 0, flags 0x0

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  79.5K     512  48.0K    6.25  DMU dnode
         1    1    16K    512     8K     512    512  100.00  other ZAP
         2    8    16K   128K   439K      1K  4.00E    0.00  other uint64[]
         4    1    16K     2K     8K     512     2K  100.00  other ZAP
         5    2    16K    16K  24.0K      1K    32K  100.00  other ZAP
        32    1    16K    512      0   7.50K    512  100.00  other ZAP
        64    1     4K    32K      0   14.5K    32K    0.00  other uint64[]

Dataset ztest/temp_4@0 [OTHER], ID 96, cr_txg 77, 71.7K, 5 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  56.0K     512    16K   15.62  DMU dnode
         1    1    16K    512     8K     512    512  100.00  other ZAP
         2    1     8K     2K      0      2K     2K    0.00  other uint64[]
         6    1     4K   128K      0      2K   128K    0.00  other uint64[]
        10    1    16K    512      0      1K    512    0.00  other uint64[]
        12    1    16K    32K      0     512    32K    0.00  other uint64[]

Dataset ztest/temp_4 [OTHER], ID 93, cr_txg 75, 71.7K, 5 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  56.0K     512    16K   15.62  DMU dnode
         1    1    16K    512     8K     512    512  100.00  other ZAP
         2    1     8K     2K      0      2K     2K    0.00  other uint64[]
         6    1     4K   128K      0      2K   128K    0.00  other uint64[]
        10    1    16K    512      0      1K    512    0.00  other uint64[]
        12    1    16K    32K      0     512    32K    0.00  other uint64[]

Dataset ztest/ds_1 [OTHER], ID 18, cr_txg 63, 928K, 9 objects

    ZIL header: claim_txg 0, claim_blk_seq 0, claim_lr_seq 0 replay_seq 0, flags 0x0

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  87.5K     512   112K    4.02  DMU dnode
         1    1    16K     1K     8K     512     1K  100.00  other ZAP
         2    2    16K    16K  24.0K   1.50K    32K  100.00  other ZAP
         5    1    16K  2.50K     8K   1.50K  2.50K  100.00  other ZAP
         8    8     4K   128K    64K     512   834T    0.00  other uint64[] (K=sha256) (Z=gzip-5)
         9    7     4K    16K    64K     512  2.24T    0.00  other uint64[]
        10    1     4K   128K     4K      1K   128K  100.00  other uint64[]
        96    9     8K    64K   638K   11.0K  2.00E    0.00  other uint64[]
       139    2    16K    16K  24.0K   6.00K    32K  100.00  other ZAP
       208    1    16K    512      0     512    512  100.00  other ZAP

Dataset ztest/ds_6 [OTHER], ID 101, cr_txg 77, 295K, 6 objects

    ZIL header: claim_txg 0, claim_blk_seq 0, claim_lr_seq 0 replay_seq 0, flags 0x0

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  79.5K     512    64K    4.69  DMU dnode
         1    1    16K    512     8K     512    512  100.00  other ZAP
         3    1     4K    16K    16K   13.0K    16K  100.00  other uint64[]
        47    6    16K   128K  95.5K      1K   871T    0.00  other uint64[] (K=fletcher4) (Z=on)
        49    6     8K     8K  87.5K     512  1.85T    0.00  other uint64[]
        79    1    16K    512      0      2K    512  100.00  other ZAP
        97    1    16K    512      0      2K    512  100.00  other ZAP

Dataset ztest/temp_9 [OTHER], ID 107, cr_txg 80, 618K, 5 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  56.0K     512    16K   15.62  DMU dnode
         1    1    16K    512     8K     512    512  100.00  other ZAP
         2    6     4K    512   136K      8K   912M    0.00  other uint64[]
        18    5     8K     2K  83.5K     512   576M    0.00  other uint64[]
        19    4    16K     4K  48.0K   1.50K   720M    0.00  other uint64[]
        22    3    16K   128K   279K      1K   848M    0.03  other uint64[]

Dataset ztest/ds_4 [OTHER], ID 81, cr_txg 73, 506K, 8 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  87.5K     512    64K    6.25  DMU dnode
         1    1    16K    512     8K     512    512  100.00  other ZAP
         2    2    16K    16K  24.0K      2K    32K  100.00  other ZAP
        32    1    16K  2.50K     8K   14.5K  2.50K  100.00  other ZAP
        61    1     8K    512      0     512    512    0.00  other uint64[]
        62    8    16K    64K   255K     512  1.00E    0.00  other uint64[]
        96    1    16K    512      0   11.5K    512  100.00  other ZAP
       119    9     4K    512    64K     512  48.7T    0.00  other uint64[]
       120    7     4K     8K  52.0K      2K  2.28T    0.00  other uint64[]

Dataset ztest/ds_3 [OTHER], ID 75, cr_txg 69, 1.26M, 17 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  87.5K     512   128K    6.64  DMU dnode
         1    1    16K  1.50K     8K     512  1.50K  100.00  other ZAP
         2    9     4K     2K  71.5K      2K  81.4T    0.00  other uint64[]
         6    5    16K     8K  36.0K   12.0K   977G    0.00  other uint64[]
        30    1     4K    64K  79.5K     512    64K  100.00  other uint64[]
        31    1    16K    512      0     512    512  100.00  other uint64[]
        65    1    16K    512      0     512    512  100.00  other ZAP
        66    9    16K    512   415K   1.50K  2.00E    0.00  other uint64[]
       224    1    16K    512      0      1K    512  100.00  other ZAP
       226    7     8K    32K   295K     512   635T    0.00  other uint64[] (K=fletcher2) (Z=zle)
       227    8     4K     1K   239K     512  1.86T    0.00  other uint64[]
       228    1     4K    64K     4K   1.50K    64K  100.00  other uint64[]
       231    1     4K     4K      0      1K     4K    0.00  other uint64[]
       233    1     8K    512      0      2K    512    0.00  other uint64[]
       237    1     4K    32K      0     512    32K    0.00  other uint64[]
       238    1    16K    64K      0   1.50K    64K    0.00  other uint64[]
       241    1    16K  1.50K  12.0K      1K  1.50K  100.00  other ZAP
       243    2    16K    16K  36.0K      2K    32K  100.00  other ZAP

Dataset ztest/temp_2 [OTHER], ID 139, cr_txg 109, 63.8K, 1 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  56.0K     512    16K    3.12  DMU dnode
         1    1    16K    512      0     512    512  100.00  other ZAP

Dataset ztest/temp_5 [OTHER], ID 148, cr_txg 116, 586K, 5 objects

    ZIL header: claim_txg 0, claim_blk_seq 0, claim_lr_seq 0 replay_seq 0, flags 0x0

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  71.5K     512    32K    7.81  DMU dnode
         1    1    16K    512      0     512    512  100.00  other ZAP
         8    5     4K     1K  91.5K    9.5K   960M    0.00  other uint64[]
        27    5     4K     8K  40.0K     512   528M    0.00  other uint64[]
        28    4     4K    64K   104K      2K   480M    0.01  other uint64[]
        32    3    16K    64K   271K     512   928M    0.02  other uint64[]

Dataset ztest/ds_0 [OTHER], ID 5, cr_txg 7, 1.45M, 14 objects

    ZIL header: claim_txg 0, claim_blk_seq 0, claim_lr_seq 0 replay_seq 0, flags 0x0

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K  71.5K     512    32K   21.88  DMU dnode
         1    1    16K     1K     8K     512     1K  100.00  other ZAP
         2    5     8K    512   108K      1K   912M    0.00  other uint64[]
         4    4    16K     4K  28.0K   3.00K   384M    0.00  other uint64[]
        10    4     4K    64K   108K      1K   864M    0.01  other uint64[]
        12    4     4K    64K      0      2K   464M    0.00  other uint64[]
        16    4     4K    32K   383K      1K  1008M    0.02  other uint64[]
        18    9    16K     8K   622K     512  4.00E    0.00  other uint64[]
        19    2    16K    16K  24.0K      2K    32K  100.00  other ZAP
        23    8     8K     8K  56.0K     512   595T    0.00  other uint64[] (K=fletcher2) (Z=gzip-7)
        24    8     4K     1K  60.0K     512  1.19T    0.00  other uint64[]
        25    1    16K     1K     8K      1K     1K  100.00  other ZAP
        27    1     4K    16K      0      2K    16K  100.00  other uint64[]
        32    1     4K     8K      0      2K     8K    0.00  other uint64[] (K=on) (Z=zle)
        36    6     4K    64K      0     512   486G    0.00  other uint64[]

Dataset ztest [ZPL], ID 51, cr_txg 1, 7.97K, 0 objects

    Object  lvl   iblk   dblk  dsize  dnsize  lsize   %full  type
         0    7    16K    16K      0     512    16K    0.00  DMU dnode

Verified large_blocks feature refcount of 0 is correct
Verified large_dnode feature refcount of 13 is correct
Verified sha512 feature refcount of 0 is correct
Verified skein feature refcount of 2 is correct
Verified edonr feature refcount of 1 is correct
Verified userobj_accounting feature refcount of 0 is correct

Traversing all blocks to verify checksums and verify nothing leaked ...

loading space map for vdev 2 of 3, metaslab 0 of 23 ....

    No leaks (block sum matches space maps exactly)

    bp count:            1062
    ganged count:          10
    bp logical:      14778368      avg:  13915
    bp physical:      5126144      avg:   4826     compression:   2.88
    bp allocated:    27193344      avg:  25605     compression:   0.54
    bp deduped:             0    ref>1:      0   deduplication:   1.00
    SPA allocated:   27193344     used:  2.35%

    additional, non-pointer bps of type 0:        173
    Dittoed blocks on same vdev: 610

Blocks  LSIZE   PSIZE   ASIZE     avg    comp   %Total  Type
     -      -       -       -       -       -        -  unallocated
     2    32K      8K   72.0K   36.0K    4.00     0.27  object directory
     -      -       -       -       -       -        -  object array
     1    16K      4K   36.0K   36.0K    4.00     0.14  packed nvlist
     -      -       -       -       -       -        -  packed nvlist size
     1    16K      4K   36.0K   36.0K    4.00     0.14  bpobj
     -      -       -       -       -       -        -  bpobj header
     -      -       -       -       -       -        -  SPA space map header
    22   124K   84.0K    756K   34.4K    1.48     2.85  SPA space map
     5  20.0K   20.0K   60.0K   12.0K    1.00     0.23  ZIL intent log
   112  1.75M    480K   3.01M   27.5K    3.73    11.61  DMU dnode
    15  30.0K   30.0K    384K   25.6K    1.00     1.45  DMU objset
     -      -       -       -       -       -        -  DSL directory
    17   9.5K   1.50K   36.0K   2.12K    6.33     0.14  DSL directory child map
     -      -       -       -       -       -        -  DSL dataset snap map
    17  8.50K     512   36.0K   2.12K   17.00     0.14  DSL props
     -      -       -       -       -       -        -  DSL dataset
     -      -       -       -       -       -        -  ZFS znode
     -      -       -       -       -       -        -  ZFS V0 ACL
     -      -       -       -       -       -        -  ZFS plain file
     -      -       -       -       -       -        -  ZFS directory
     -      -       -       -       -       -        -  ZFS master node
     -      -       -       -       -       -        -  ZFS delete queue
     -      -       -       -       -       -        -  zvol object
     -      -       -       -       -       -        -  zvol prop
     -      -       -       -       -       -        -  other uint8[]
   743  11.5M   4.12M   20.0M   27.5K    2.78    76.95  other uint64[]
    48   360K    103K    972K   20.2K    3.50     3.66  other ZAP
     -      -       -       -       -       -        -  persistent error log
     1   128K      4K   36.0K   36.0K   32.00     0.14  SPA history
     -      -       -       -       -       -        -  SPA history offsets
     -      -       -       -       -       -        -  Pool properties
     -      -       -       -       -       -        -  DSL permissions
     -      -       -       -       -       -        -  ZFS ACL
     -      -       -       -       -       -        -  ZFS SYSACL
     -      -       -       -       -       -        -  FUID table
     -      -       -       -       -       -        -  FUID table size
     1     1K      1K   36.0K   36.0K    1.00     0.14  DSL dataset next clones
     -      -       -       -       -       -        -  scan work queue
     -      -       -       -       -       -        -  ZFS user/group used
     -      -       -       -       -       -        -  ZFS user/group quota
     -      -       -       -       -       -        -  snapshot refcount tags
     2     8K      4K   36.0K   18.0K    2.00     0.14  DDT ZAP algorithm
     2    32K      8K   72.0K   36.0K    4.00     0.27  DDT statistics
     -      -       -       -       -       -        -  System attributes
     -      -       -       -       -       -        -  SA master node
     -      -       -       -       -       -        -  SA attr registration
     -      -       -       -       -       -        -  SA attr layouts
     -      -       -       -       -       -        -  scan translations
     -      -       -       -       -       -        -  deduplicated block
     -      -       -       -       -       -        -  DSL deadlist map
     -      -       -       -       -       -        -  DSL deadlist map hdr
     1     1K      1K   36.0K   36.0K    1.00     0.14  DSL dir clones
     -      -       -       -       -       -        -  bpobj subobj
     5  44.0K   20.0K    180K   36.0K    2.20     0.68  deferred free
     -      -       -       -       -       -        -  dedup ditto
    31  48.5K     16K    252K   8.13K    3.03     0.95  other
 1.04K  14.1M   4.89M   25.9M   25.0K    2.88   100.00  Total

                            capacity   operations   bandwidth  ---- errors ----
description                used avail  read write  read write  read write cksum
ztest                     25.9M 1.05G 6.58K     0 41.4M     0     0     0     0
  raidz2                  22.3M  346M 5.50K     0 30.7M     0     0     0     0
    /var/tmp/ztest.0a                 1.68K     0 9.46M     0     0     0     0
    /var/tmp/ztest.1a                 1.59K     0 8.94M     0     0     0    34
    /var/tmp/ztest.2a                 2.23K     0 12.3M     0     0     0     0
  raidz2                  1.63M  366M   642     0 5.19M     0     0     0     0
    /var/tmp/ztest.3a                   195     0 1.65M     0     0     0     0
    /var/tmp/ztest.4a                   195     0 1.65M     0     0     0     0
    /var/tmp/ztest.5a                   252     0 1.89M     0     0     0     0
  raidz2                  1.96M  366M   470     0 5.44M     0     0     0     0
    /var/tmp/ztest.6a                   139     0 1.66M     0     0     0     0
    /var/tmp/ztest.7a                   139     0 1.66M     0     0     0     0
    /var/tmp/ztest.8a                   192     0 2.11M     0     0     0     0
jsalinasintel commented 7 years ago

I still see this under a newer build built on Tue Dec 20th 2016: Git Build Data Revision: a3823f428deb8f15dd383dc6210b2e16926f4217 refs/remotes/origin/master Built Branches refs/remotes/origin/master: Build #186 of Revision a3823f428deb8f15dd383dc6210b2e16926f4217 (refs/remotes/origin/master)

zap_update(os, object, name, wsize, wc, data, tx) == 0 ASSERT at ztest.c:4751:ztest_zap_parallel()/usr/sbin/ztest[0x40a7cf] /lib64/libpthread.so.0(+0xf100)[0x7f436e5f3100] /lib64/libc.so.6(gsignal+0x37)[0x7f436e2575f7] /lib64/libc.so.6(abort+0x148)[0x7f436e258ce8] /usr/sbin/ztest[0x4089dd] /usr/sbin/ztest[0x412272] /usr/sbin/ztest[0x40cda3] /lib64/libzpool.so.2(zk_thread_helper+0x1ec)[0x7f436f7e796c] /lib64/libpthread.so.0(+0x7dc5)[0x7f436e5ebdc5] /lib64/libc.so.6(clone+0x6d)[0x7f436e318ced] child died with signal 6

[Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/usr/sbin/ztest'. Program terminated with signal 6, Aborted.

0 0x00007f436e2575f7 in raise () from /lib64/libc.so.6

*

behlendorf commented 5 years ago

Closing. This has been non-reproducible as of d8244d34bdc3 after 100's and 100's of hours or ztest(1) run time. We'll open new issues for any remaining ztest(1) failures which are observed.