search

openzfs / zfs

OpenZFS on Linux and FreeBSD
https://openzfs.github.io/openzfs-docs
Other
10.55k stars 1.74k forks source link

PANIC: MySQL parameter innodb_flush_method set to O_DIRECT (changed from fsync) #16642

Open drago72 opened 3 days ago

drago72 commented 3 days ago

System information

Type Version/Name
Distribution Name Ubuntu
Distribution Version 22.04
Kernel Version 6.10.14
Architecture x86_64
OpenZFS Version 2.3.99-1 (master up to commit c642e985e596eb38d9ea9be2bc97ff0fd9c7d5a0)

Describe the problem you're observing

Using MySQL with innodb_flush_method = fsync doesn't cause any issues. Switching the parameter to O_DIRECT and running an intensive test resulted in a kernel panic.

(I don't have extensive expertise in C or English, but I'd like to contribute to verifying filesystem stability before the final release of version 2.3. Thanks to all project contributors for their work.)

Describe how to reproduce the problem

Switched MySQL configuration innodb_flush_method from fsync to O_DIRECT and set dataset parameter direct = standard

Include any warning/errors/backtraces from the system logs

$ uname -r
6.10.14-ful01

$ zpool -V
zfs-2.3.99-1
zfs-kmod-2.3.99-1

$ arc_summary
------------------------------------------------------------------------
ZFS Subsystem Report                            Sun Oct 13 08:30:15 2024
Linux 6.10.14-ful01                                             2.3.99-1
Machine: drago-server-ubuntu (x86_64)                           2.3.99-1

[...]

Solaris Porting Layer (SPL):
        spl_hostid                                                     0
        spl_hostid_path                                      /etc/hostid
        spl_kmem_alloc_max                                       1048576
        spl_kmem_alloc_warn                                        65536
        spl_kmem_cache_kmem_threads                                    4
        spl_kmem_cache_magazine_size                                   0
        spl_kmem_cache_max_size                                       32
        spl_kmem_cache_obj_per_slab                                    8
        spl_kmem_cache_slab_limit                                  16384
        spl_panic_halt                                                 0
        spl_schedule_hrtimeout_slack_us                                0
        spl_taskq_kick                                                 0
        spl_taskq_thread_bind                                          0
        spl_taskq_thread_dynamic                                       1
        spl_taskq_thread_priority                                      1
        spl_taskq_thread_sequential                                    4
        spl_taskq_thread_timeout_ms                                 5000

Tunables:
        brt_zap_default_bs                                            12
        brt_zap_default_ibs                                           12
        brt_zap_prefetch                                               1
        dbuf_cache_hiwater_pct                                        10
        dbuf_cache_lowater_pct                                        10
        dbuf_cache_max_bytes                        18446744073709551615
        dbuf_cache_shift                                               5
        dbuf_metadata_cache_max_bytes               18446744073709551615
        dbuf_metadata_cache_shift                                      6
        dbuf_mutex_cache_shift                                         0
        ddt_zap_default_bs                                            15
        ddt_zap_default_ibs                                           15
        dmu_ddt_copies                                                 0
        dmu_object_alloc_chunk_shift                                   7
        dmu_prefetch_max                                       134217728
        icp_aes_impl                cycle [fastest] generic x86_64 aesni
        icp_gcm_avx_chunk_size                                     32736
        icp_gcm_impl               cycle [fastest] avx generic pclmulqdq
        ignore_hole_birth                                              1
        l2arc_exclude_special                                          0
        l2arc_feed_again                                               1
        l2arc_feed_min_ms                                            200
        l2arc_feed_secs                                                1
        l2arc_headroom                                                 8
        l2arc_headroom_boost                                         200
        l2arc_meta_percent                                            50
        l2arc_mfuonly                                                  2
        l2arc_noprefetch                                               1
        l2arc_norw                                                     0
        l2arc_rebuild_blocks_min_l2size                       1073741824
        l2arc_rebuild_enabled                                          1
        l2arc_trim_ahead                                               0
        l2arc_write_boost                                       33554432
        l2arc_write_max                                         33554432
        metaslab_aliquot                                         1048576
        metaslab_bias_enabled                                          1
        metaslab_debug_load                                            0
        metaslab_debug_unload                                          0
        metaslab_df_max_search                                  16777216
        metaslab_df_use_largest_segment                                0
        metaslab_force_ganging                                  16777217
        metaslab_force_ganging_pct                                     3
        metaslab_fragmentation_factor_enabled                          1
        metaslab_lba_weighting_enabled                                 1
        metaslab_preload_enabled                                       1
        metaslab_preload_limit                                        10
        metaslab_preload_pct                                          50
        metaslab_unload_delay                                         32
        metaslab_unload_delay_ms                                  600000
        raidz_expand_max_copy_bytes                            167772160
        raidz_expand_max_reflow_bytes                                  0
        raidz_io_aggregate_rows                                        4
        send_holes_without_birth_time                                  1
        spa_asize_inflation                                           24
        spa_config_path                             /etc/zfs/zpool.cache
        spa_cpus_per_allocator                                         4
        spa_load_print_vdev_tree                                       0
        spa_load_verify_data                                           1
        spa_load_verify_metadata                                       1
        spa_load_verify_shift                                          4
        spa_num_allocators                                             4
        spa_slop_shift                                                 5
        spa_upgrade_errlog_limit                                       0
        vdev_file_logical_ashift                                       9
        vdev_file_physical_ashift                                      9
        vdev_removal_max_span                                      32768
        vdev_validate_skip                                             0
        zap_iterate_prefetch                                           1
        zap_micro_max_size                                        131072
        zap_shrink_enabled                                             1
        zfetch_hole_shift                                              2
        zfetch_max_distance                                     67108864
        zfetch_max_idistance                                    67108864
        zfetch_max_reorder                                      16777216
        zfetch_max_sec_reap                                            2
        zfetch_max_streams                                             8
        zfetch_min_distance                                      4194304
        zfetch_min_sec_reap                                            1
        zfs_abd_scatter_enabled                                        1
        zfs_abd_scatter_max_order                                      9
        zfs_abd_scatter_min_size                                    1536
        zfs_active_allocator                                     dynamic
        zfs_admin_snapshot                                             0
        zfs_allow_redacted_dataset_mount                               0
        zfs_arc_average_blocksize                                   8192
        zfs_arc_dnode_limit                                            0
        zfs_arc_dnode_limit_percent                                   10
        zfs_arc_dnode_reduce_percent                                  10
        zfs_arc_evict_batch_limit                                     10
        zfs_arc_eviction_pct                                         200
        zfs_arc_grow_retry                                             0
        zfs_arc_lotsfree_percent                                      10
        zfs_arc_max                                          17179869184
        zfs_arc_meta_balance                                         500
        zfs_arc_min                                           8589934592
        zfs_arc_min_prefetch_ms                                        0
        zfs_arc_min_prescient_prefetch_ms                              0
        zfs_arc_pc_percent                                             0
        zfs_arc_prune_task_threads                                     1
        zfs_arc_shrink_shift                                           0
        zfs_arc_shrinker_limit                                     10000
        zfs_arc_shrinker_seeks                                         2
        zfs_arc_sys_free                                      1073741824
        zfs_async_block_max_blocks                  18446744073709551615
        zfs_autoimport_disable                                         1
        zfs_bclone_enabled                                             1
        zfs_bclone_wait_dirty                                          0
        zfs_blake3_impl               cycle [fastest] generic sse2 sse41
        zfs_btree_verify_intensity                                     0
        zfs_checksum_events_per_second                                20
        zfs_commit_timeout_pct                                        10
        zfs_compressed_arc_enabled                                     1
        zfs_condense_indirect_commit_entry_delay_ms                    0
        zfs_condense_indirect_obsolete_pct                            25
        zfs_condense_indirect_vdevs_enable                             1
        zfs_condense_max_obsolete_bytes                       1073741824
        zfs_condense_min_mapping_bytes                            131072
        zfs_dbgmsg_enable                                              1
        zfs_dbgmsg_maxsize                                       4194304
        zfs_dbuf_state_index                                           0
        zfs_ddt_data_is_special                                        1
        zfs_deadman_checktime_ms                                   60000
        zfs_deadman_enabled                                            1
        zfs_deadman_events_per_second                                  1
        zfs_deadman_failmode                                        wait
        zfs_deadman_synctime_ms                                   600000
        zfs_deadman_ziotime_ms                                    300000
        zfs_dedup_log_flush_entries_min                             1000
        zfs_dedup_log_flush_flow_rate_txgs                            10
        zfs_dedup_log_flush_min_time_ms                             1000
        zfs_dedup_log_flush_passes_max                                 8
        zfs_dedup_log_mem_max                                  333422592
        zfs_dedup_log_mem_max_percent                                  1
        zfs_dedup_log_txg_max                                          8
        zfs_dedup_prefetch                                             0
        zfs_default_bs                                                 9
        zfs_default_ibs                                               17
        zfs_delay_min_dirty_percent                                   60
        zfs_delay_scale                                           500000
        zfs_delete_blocks                                          20480
        zfs_dio_enabled                                                0
        zfs_dio_write_verify_events_per_second                        20
        zfs_dirty_data_max                                    3334225920
        zfs_dirty_data_max_max                                4294967296
        zfs_dirty_data_max_max_percent                                25
        zfs_dirty_data_max_percent                                    10
        zfs_dirty_data_sync_percent                                   20
        zfs_disable_ivset_guid_check                                   0
        zfs_dmu_offset_next_sync                                       1
        zfs_embedded_slog_min_ms                                      64
        zfs_expire_snapshot                                          300
        zfs_fallocate_reserve_percent                                110
        zfs_flags                                                      0
        zfs_fletcher_4_impl [fastest] scalar superscalar superscalar4 sse2 ssse3
        zfs_free_bpobj_enabled                                         1
        zfs_free_leak_on_eio                                           0
        zfs_free_min_time_ms                                        1000
        zfs_history_output_max                                   1048576
        zfs_immediate_write_sz                                     32768
        zfs_initialize_chunk_size                                1048576
        zfs_initialize_value                        16045690984833335022
        zfs_keep_log_spacemaps_at_export                               0
        zfs_key_max_salt_uses                                  400000000
        zfs_livelist_condense_new_alloc                                0
        zfs_livelist_condense_sync_cancel                              0
        zfs_livelist_condense_sync_pause                               0
        zfs_livelist_condense_zthr_cancel                              0
        zfs_livelist_condense_zthr_pause                               0
        zfs_livelist_max_entries                                  500000
        zfs_livelist_min_percent_shared                               75
        zfs_lua_max_instrlimit                                 100000000
        zfs_lua_max_memlimit                                   104857600
        zfs_max_async_dedup_frees                                 100000
        zfs_max_dataset_nesting                                       50
        zfs_max_log_walking                                            5
        zfs_max_logsm_summary_length                                  10
        zfs_max_missing_tvds                                           0
        zfs_max_nvlist_src_size                                        0
        zfs_max_recordsize                                      16777216
        zfs_metaslab_find_max_tries                                  100
        zfs_metaslab_fragmentation_threshold                          70
        zfs_metaslab_max_size_cache_sec                             3600
        zfs_metaslab_mem_limit                                        25
        zfs_metaslab_segment_weight_enabled                            1
        zfs_metaslab_switch_threshold                                  2
        zfs_metaslab_try_hard_before_gang                              0
        zfs_mg_fragmentation_threshold                                95
        zfs_mg_noalloc_threshold                                       0
        zfs_min_metaslabs_to_flush                                     1
        zfs_multihost_fail_intervals                                  10
        zfs_multihost_history                                          0
        zfs_multihost_import_intervals                                20
        zfs_multihost_interval                                      1000
        zfs_multilist_num_sublists                                     0
        zfs_no_scrub_io                                                0
        zfs_no_scrub_prefetch                                          0
        zfs_nocacheflush                                               0
        zfs_nopwrite_enabled                                           1
        zfs_object_mutex_size                                         64
        zfs_obsolete_min_time_ms                                     500
        zfs_override_estimate_recordsize                               0
        zfs_pd_bytes_max                                        52428800
        zfs_per_txg_dirty_frees_percent                               30
        zfs_prefetch_disable                                           0
        zfs_read_history                                               0
        zfs_read_history_hits                                          0
        zfs_rebuild_max_segment                                  1048576
        zfs_rebuild_scrub_enabled                                      1
        zfs_rebuild_vdev_limit                                  67108864
        zfs_reconstruct_indirect_combinations_max                   4096
        zfs_recover                                                    0
        zfs_recv_best_effort_corrective                                0
        zfs_recv_queue_ff                                             20
        zfs_recv_queue_length                                   16777216
        zfs_recv_write_batch_size                                1048576
        zfs_removal_ignore_errors                                      0
        zfs_removal_suspend_progress                                   0
        zfs_remove_max_segment                                  16777216
        zfs_resilver_defer_percent                                    10
        zfs_resilver_disable_defer                                     0
        zfs_resilver_min_time_ms                                    3000
        zfs_scan_blkstats                                              0
        zfs_scan_checkpoint_intval                                  7200
        zfs_scan_fill_weight                                           3
        zfs_scan_ignore_errors                                         0
        zfs_scan_issue_strategy                                        0
        zfs_scan_legacy                                                0
        zfs_scan_max_ext_gap                                     2097152
        zfs_scan_mem_lim_fact                                         20
        zfs_scan_mem_lim_soft_fact                                    20
        zfs_scan_report_txgs                                           0
        zfs_scan_strict_mem_lim                                        0
        zfs_scan_suspend_progress                                      0
        zfs_scan_vdev_limit                                     16777216
        zfs_scrub_after_expand                                         1
        zfs_scrub_error_blocks_per_txg                              4096
        zfs_scrub_min_time_ms                                       1000
        zfs_send_corrupt_data                                          0
        zfs_send_no_prefetch_queue_ff                                 20
        zfs_send_no_prefetch_queue_length                        1048576
        zfs_send_queue_ff                                             20
        zfs_send_queue_length                                   16777216
        zfs_send_unmodified_spill_blocks                               1
        zfs_sha256_impl            cycle [fastest] generic x64 ssse3 avx
        zfs_sha512_impl                  cycle [fastest] generic x64 avx
        zfs_slow_io_events_per_second                                 20
        zfs_snapshot_history_enabled                                   1
        zfs_snapshot_no_setuid                                         0
        zfs_spa_discard_memory_limit                            16777216
        zfs_special_class_metadata_reserve_pct                        25
        zfs_sync_pass_deferred_free                                    2
        zfs_sync_pass_dont_compress                                    8
        zfs_sync_pass_rewrite                                          2
        zfs_traverse_indirect_prefetch_limit                          32
        zfs_trim_extent_bytes_max                              134217728
        zfs_trim_extent_bytes_min                                  32768
        zfs_trim_metaslab_skip                                         0
        zfs_trim_queue_limit                                          10
        zfs_trim_txg_batch                                            32
        zfs_txg_history                                              100
        zfs_txg_timeout                                                7
        zfs_unflushed_log_block_max                               131072
        zfs_unflushed_log_block_min                                 1000
        zfs_unflushed_log_block_pct                                  400
        zfs_unflushed_log_txg_max                                   1000
        zfs_unflushed_max_mem_amt                             1073741824
        zfs_unflushed_max_mem_ppm                                   1000
        zfs_unlink_suspend_progress                                    0
        zfs_user_indirect_is_special                                   1
        zfs_vdev_aggregation_limit                                524288
        zfs_vdev_aggregation_limit_non_rotating                   131072
        zfs_vdev_async_read_max_active                                 8
        zfs_vdev_async_read_min_active                                 4
        zfs_vdev_async_write_active_max_dirty_percent                 60
        zfs_vdev_async_write_active_min_dirty_percent                 30
        zfs_vdev_async_write_max_active                                8
        zfs_vdev_async_write_min_active                                4
        zfs_vdev_def_queue_depth                                      32
        zfs_vdev_default_ms_count                                    200
        zfs_vdev_default_ms_shift                                     29
        zfs_vdev_direct_write_verify                                   1
        zfs_vdev_disk_classic                                          0
        zfs_vdev_disk_max_segs                                         0
        zfs_vdev_failfast_mask                                         1
        zfs_vdev_initializing_max_active                               1
        zfs_vdev_initializing_min_active                               1
        zfs_vdev_max_active                                         1000
        zfs_vdev_max_auto_ashift                                      14
        zfs_vdev_max_ms_shift                                         34
        zfs_vdev_min_auto_ashift                                       9
        zfs_vdev_min_ms_count                                         16
        zfs_vdev_mirror_non_rotating_inc                               0
        zfs_vdev_mirror_non_rotating_seek_inc                          1
        zfs_vdev_mirror_rotating_inc                                   0
        zfs_vdev_mirror_rotating_seek_inc                              5
        zfs_vdev_mirror_rotating_seek_offset                     1048576
        zfs_vdev_ms_count_limit                                   131072
        zfs_vdev_nia_credit                                            5
        zfs_vdev_nia_delay                                             5
        zfs_vdev_open_timeout_ms                                    1000
        zfs_vdev_queue_depth_pct                                    1000
        zfs_vdev_raidz_impl   cycle [fastest] original scalar sse2 ssse3
        zfs_vdev_read_gap_limit                                    32768
        zfs_vdev_rebuild_max_active                                    2
        zfs_vdev_rebuild_min_active                                    1
        zfs_vdev_removal_max_active                                    2
        zfs_vdev_removal_min_active                                    1
        zfs_vdev_scheduler                                        unused
        zfs_vdev_scrub_max_active                                      2
        zfs_vdev_scrub_min_active                                      1
        zfs_vdev_sync_read_max_active                                  8
        zfs_vdev_sync_read_min_active                                  4
        zfs_vdev_sync_write_max_active                                 8
        zfs_vdev_sync_write_min_active                                 4
        zfs_vdev_trim_max_active                                       2
        zfs_vdev_trim_min_active                                       1
        zfs_vdev_write_gap_limit                                    4096
        zfs_vnops_read_chunk_size                                1048576
        zfs_wrlog_data_max                                    6668451840
        zfs_xattr_compat                                               0
        zfs_zevent_len_max                                           512
        zfs_zevent_retain_expire_secs                                900
        zfs_zevent_retain_max                                       2000
        zfs_zil_clean_taskq_maxalloc                             1048576
        zfs_zil_clean_taskq_minalloc                                1024
        zfs_zil_clean_taskq_nthr_pct                                 100
        zfs_zil_saxattr                                                1
        zil_maxblocksize                                          131072
        zil_maxcopied                                               7680
        zil_nocacheflush                                               0
        zil_replay_disable                                             0
        zil_slog_bulk                                           67108864
        zio_deadman_log_all                                            0
        zio_dva_throttle_enabled                                       1
        zio_requeue_io_start_cut_in_line                               1
        zio_slow_io_ms                                             30000
        zio_taskq_batch_pct                                           80
        zio_taskq_batch_tpq                                            0
        zio_taskq_read                         fixed,1,8 null scale null
        zio_taskq_write                             sync null scale null
        zio_taskq_write_tpq                                           16
        zstd_abort_size                                           131072
        zstd_earlyabort_pass                                           1
        zvol_blk_mq_blocks_per_thread                                  8
        zvol_blk_mq_queue_depth                                      128
        zvol_enforce_quotas                                            1
        zvol_inhibit_dev                                               0
        zvol_major                                                   230
        zvol_max_discard_blocks                                    16384
        zvol_num_taskqs                                                0
        zvol_open_timeout_ms                                        1000
        zvol_prefetch_bytes                                       131072
        zvol_request_sync                                              0
        zvol_threads                                                   0
        zvol_use_blk_mq                                                1
        zvol_volmode                                                   1

$ zpool status zfs1
pool: zfs1
 state: ONLINE
status: Some supported and requested features are not enabled on the pool.
    The pool can still be used, but some features are unavailable.
action: Enable all features using 'zpool upgrade'. Once this is done,
    the pool may no longer be accessible by software that does not support
    the features. See zpool-features(7) for details.
  scan: scrub repaired 0B in 20:42:05 with 0 errors on Wed Oct  9 09:26:34 2024
config:

    NAME                 STATE     READ WRITE CKSUM
    zfs1                 ONLINE       0     0     0
      draid1:3d:4c:0s-0  ONLINE       0     0     0
        sde4             ONLINE       0     0     0
        sdc2             ONLINE       0     0     0
        sdd2             ONLINE       0     0     0
        sdb2             ONLINE       0     0     0
    cache
      sda6               ONLINE       0     0     0

errors: No known data errors

$ zpool get all zfs1
NAME  PROPERTY                       VALUE                          SOURCE
zfs1  size                           7.31T                          -
zfs1  capacity                       37%                            -
zfs1  altroot                        -                              default
zfs1  health                         ONLINE                         -
zfs1  guid                           14708664622803853334           -
zfs1  version                        -                              default
zfs1  bootfs                         -                              default
zfs1  delegation                     on                             default
zfs1  autoreplace                    off                            default
zfs1  cachefile                      -                              default
zfs1  failmode                       wait                           default
zfs1  listsnapshots                  off                            default
zfs1  autoexpand                     off                            default
zfs1  dedupratio                     1.93x                          -
zfs1  free                           4.58T                          -
zfs1  allocated                      2.73T                          -
zfs1  readonly                       off                            -
zfs1  ashift                         12                             local
zfs1  comment                        -                              default
zfs1  expandsize                     -                              -
zfs1  freeing                        0                              -
zfs1  fragmentation                  34%                            -
zfs1  leaked                         0                              -
zfs1  multihost                      off                            default
zfs1  checkpoint                     -                              -
zfs1  load_guid                      15650802406476463967           -
zfs1  autotrim                       off                            default
zfs1  compatibility                  off                            default
zfs1  bcloneused                     771M                           -
zfs1  bclonesaved                    2.49G                          -
zfs1  bcloneratio                    4.30x                          -
zfs1  dedup_table_size               15.9E                          -
zfs1  dedup_table_quota              auto                           default
zfs1  feature@async_destroy          enabled                        local
[... all enabled]
zfs1  feature@fast_dedup             enabled                        local
zfs1  feature@longname               disabled                       local
zfs1  feature@large_microzap         disabled                       local

$ zfs get all zfs1/database_8.4_copy
NAME                    PROPERTY              VALUE                     SOURCE
zfs1/database_8.4_copy  type                  filesystem                -
zfs1/database_8.4_copy  creation              sab ott 12 18:29 2024     -
zfs1/database_8.4_copy  used                  307G                      -
zfs1/database_8.4_copy  available             2.62T                     -
zfs1/database_8.4_copy  referenced            304G                      -
zfs1/database_8.4_copy  compressratio         2.52x                     -
zfs1/database_8.4_copy  mounted               yes                       -
zfs1/database_8.4_copy  quota                 none                      default
zfs1/database_8.4_copy  reservation           none                      default
zfs1/database_8.4_copy  recordsize            128K                      default
zfs1/database_8.4_copy  mountpoint            /zfs1/database_8.4_copy   inherited from zfs1
zfs1/database_8.4_copy  sharenfs              off                       default
zfs1/database_8.4_copy  checksum              fletcher4                 inherited from zfs1
zfs1/database_8.4_copy  compression           lz4                       inherited from zfs1
zfs1/database_8.4_copy  atime                 off                       inherited from zfs1
zfs1/database_8.4_copy  devices               on                        default
zfs1/database_8.4_copy  exec                  on                        default
zfs1/database_8.4_copy  setuid                on                        default
zfs1/database_8.4_copy  readonly              off                       default
zfs1/database_8.4_copy  zoned                 off                       default
zfs1/database_8.4_copy  snapdir               hidden                    inherited from zfs1
zfs1/database_8.4_copy  aclmode               discard                   default
zfs1/database_8.4_copy  aclinherit            restricted                default
zfs1/database_8.4_copy  createtxg             4098543                   -
zfs1/database_8.4_copy  canmount              on                        default
zfs1/database_8.4_copy  xattr                 on                        inherited from zfs1
zfs1/database_8.4_copy  copies                1                         default
zfs1/database_8.4_copy  version               5                         -
zfs1/database_8.4_copy  utf8only              off                       -
zfs1/database_8.4_copy  normalization         none                      -
zfs1/database_8.4_copy  casesensitivity       sensitive                 -
zfs1/database_8.4_copy  vscan                 off                       default
zfs1/database_8.4_copy  nbmand                off                       default
zfs1/database_8.4_copy  sharesmb              off                       default
zfs1/database_8.4_copy  refquota              none                      default
zfs1/database_8.4_copy  refreservation        none                      default
zfs1/database_8.4_copy  guid                  178552650196415616        -
zfs1/database_8.4_copy  primarycache          all                       default
zfs1/database_8.4_copy  secondarycache        all                       local
zfs1/database_8.4_copy  usedbysnapshots       2.82G                     -
zfs1/database_8.4_copy  usedbydataset         304G                      -
zfs1/database_8.4_copy  usedbychildren        0B                        -
zfs1/database_8.4_copy  usedbyrefreservation  0B                        -
zfs1/database_8.4_copy  logbias               latency                   default
zfs1/database_8.4_copy  objsetid              8339                      -
zfs1/database_8.4_copy  dedup                 off                       default
zfs1/database_8.4_copy  mlslabel              none                      default
zfs1/database_8.4_copy  sync                  disabled                  local
zfs1/database_8.4_copy  dnodesize             auto                      inherited from zfs1
zfs1/database_8.4_copy  refcompressratio      2.53x                     -
zfs1/database_8.4_copy  written               2.83G                     -
zfs1/database_8.4_copy  logicalused           756G                      -
zfs1/database_8.4_copy  logicalreferenced     752G                      -
zfs1/database_8.4_copy  volmode               default                   default
zfs1/database_8.4_copy  filesystem_limit      none                      default
zfs1/database_8.4_copy  snapshot_limit        none                      default
zfs1/database_8.4_copy  filesystem_count      none                      default
zfs1/database_8.4_copy  snapshot_count        none                      default
zfs1/database_8.4_copy  snapdev               hidden                    default
zfs1/database_8.4_copy  acltype               posix                     inherited from zfs1
zfs1/database_8.4_copy  context               none                      default
zfs1/database_8.4_copy  fscontext             none                      default
zfs1/database_8.4_copy  defcontext            none                      default
zfs1/database_8.4_copy  rootcontext           none                      default
zfs1/database_8.4_copy  relatime              on                        default
zfs1/database_8.4_copy  redundant_metadata    most                      local
zfs1/database_8.4_copy  overlay               on                        default
zfs1/database_8.4_copy  encryption            off                       default
zfs1/database_8.4_copy  keylocation           none                      default
zfs1/database_8.4_copy  keyformat             none                      default
zfs1/database_8.4_copy  pbkdf2iters           0                         default
zfs1/database_8.4_copy  special_small_blocks  0                         default
zfs1/database_8.4_copy  snapshots_changed     sab ott 12 19:52:02 2024  -
zfs1/database_8.4_copy  prefetch              metadata                  local
zfs1/database_8.4_copy  direct                standard                  local
zfs1/database_8.4_copy  longname              off                       default

### kernel.log

Oct 13 05:43:28 drago-server-ubuntu kernel: [    6.494965] ZFS: Loaded module v2.3.99-1, ZFS pool version 5000, ZFS filesystem version 5
Oct 13 05:43:28 drago-server-ubuntu kernel: [   16.276409]  zd0: p1 p2 p3
Oct 13 05:43:28 drago-server-ubuntu kernel: [   16.568024]  zd32: p1 p2
Oct 13 05:43:28 drago-server-ubuntu kernel: [   16.616085]  zd48: p1 p2 p3 p4
Oct 13 05:43:28 drago-server-ubuntu kernel: [   16.710566]  zd64: p1 p9
Oct 13 05:43:28 drago-server-ubuntu kernel: [   16.749669]  zd80: p1 p2
Oct 13 05:43:28 drago-server-ubuntu kernel: [   16.807920]  zd96: p1 p2
[...]
Oct 13 05:43:28 drago-server-ubuntu kernel: [   18.765756] zram: Added device: zram0
Oct 13 05:43:28 drago-server-ubuntu kernel: [   18.772788] zram0: detected capacity change from 0 to 16777216
Oct 13 05:43:28 drago-server-ubuntu kernel: [   18.784882] Adding 8388604k swap on /dev/zram0.  Priority:10 extents:1 across:8388604k SS
[...]
Oct 13 06:47:43 drago-server-ubuntu kernel: [ 3873.201380] perf: interrupt took too long (3204 > 3185), lowering kernel.perf_event_max_sample_rate to 62250
[...]
Oct 13 08:11:29 drago-server-ubuntu kernel: [ 8899.410167] perf: interrupt took too long (4119 > 4005), lowering kernel.perf_event_max_sample_rate to 48500
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591384] BUG: Bad page state in process connection  pfn:7da6cd
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591391] page: refcount:0 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x7da6cd
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591394] flags: 0x17ffffc6002204(referenced|workingset|reserved|node=0|zone=2|lastcpupid=0x1fffff)
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591398] raw: 0017ffffc6002204 fffff55b1f69b348 fffff55b1f69b348 0000000000000000
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591400] raw: 0000000000000000 0000000000000000 00000000ffffffff 0000000000000000
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591401] page dumped because: PAGE_FLAGS_CHECK_AT_FREE flag(s) set
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591401] Modules linked in: vhost_net vhost vhost_iotlb macvtap macvlan tap nf_conntrack_netlink nfnetlink_acct udp_diag tcp_diag inet_diag xt_recent wireguard curve25519_x86_64 libchacha20poly1305 chacha_x86_64 poly1305_x86_64 libcurve25519_generic libchacha ip6_udp_tunnel udp_tunnel xt_CHECKSUM xt_MASQUERADE nft_chain_nat nf_nat bridge stp llc zram ip6t_REJECT nf_reject_ipv6 blocklayoutdriver xt_hl ip6t_rt ipt_REJECT nf_reject_ipv4 xt_LOG nf_log_syslog xt_comment xt_multiport nft_limit xt_limit xt_addrtype nfsv4 xt_tcpudp xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nfs nft_compat netfs nf_tables nfnetlink binfmt_misc intel_rapl_msr intel_rapl_common mei_pxp mei_hdcp x86_pkg_temp_thermal intel_powerclamp kvm_intel kvm crct10dif_pclmul polyval_clmulni polyval_generic ghash_clmulni_intel sha256_ssse3 sha1_ssse3 aesni_intel crypto_simd cryptd rapl intel_cstate snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_codec_generic snd_hda_scodec_component snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591451]  nls_iso8859_1 snd_hda_codec snd_hda_core snd_hwdep zfs(POE) snd_pcm snd_seq_midi snd_seq_midi_event snd_rawmidi snd_seq spl(OE) snd_seq_device input_leds snd_timer joydev snd at24 soundcore mei_me mei intel_smartconnect mac_hid sch_fq_codel nct6775 nct6775_core hwmon_vid nfsd coretemp msr auth_rpcgss parport_pc nfs_acl lockd ppdev grace lp parport efi_pstore sunrpc ip_tables x_tables autofs4 raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx raid1 raid0 lz4 lz4_compress hid_generic usbhid hid i915 drm_buddy i2c_algo_bit ttm drm_display_helper cec rc_core i2c_i801 i2c_mux drm_kms_helper crc32_pclmul i2c_smbus ahci r8169 libahci drm realtek xhci_pci lpc_ich xhci_pci_renesas video wmi
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591498] CPU: 1 PID: 84119 Comm: connection Tainted: P           OE      6.10.14-ful01 #1
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591500] Hardware name: To Be Filled By O.E.M. To Be Filled By O.E.M./Z77 Pro3, BIOS P2.10 07/12/2013
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591502] Call Trace:
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591504]  <TASK>
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591507]  dump_stack_lvl+0x76/0xa0
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591511]  dump_stack+0x10/0x20
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591513]  bad_page+0x71/0x110
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591516]  free_page_is_bad_report+0x86/0xa0
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591519]  free_unref_page+0x34c/0x520
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591522]  __folio_put+0x9e/0xe0
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591526]  zfs_uio_free_dio_pages+0x6e/0x180 [zfs]
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591755]  zfs_write+0xc0c/0xe70 [zfs]
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.591975]  zpl_iter_write+0x119/0x1b0 [zfs]
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592174]  vfs_write+0x262/0x450
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592179]  __x64_sys_pwrite64+0xa6/0xd0
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592182]  x64_sys_call+0x1e45/0x25c0
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592185]  do_syscall_64+0x70/0x130
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592188]  ? __count_memcg_events+0x76/0x120
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592192]  ? count_memcg_events.constprop.0+0x2a/0x50
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592196]  ? handle_mm_fault+0xaf/0x330
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592198]  ? do_user_addr_fault+0x340/0x6c0
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592202]  ? irqentry_exit_to_user_mode+0x56/0x180
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592205]  ? irqentry_exit+0x43/0x50
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592207]  ? exc_page_fault+0x93/0x1b0
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592209]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592213] RIP: 0033:0x7fd9a4f1283f
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592235] Code: 08 89 3c 24 48 89 4c 24 18 e8 cd e1 f7 ff 4c 8b 54 24 18 48 8b 54 24 10 41 89 c0 48 8b 74 24 08 8b 3c 24 b8 12 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 31 44 89 c7 48 89 04 24 e8 0d e2 f7 ff 48 8b
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592237] RSP: 002b:00007fd9523f11f0 EFLAGS: 00000293 ORIG_RAX: 0000000000000012
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592240] RAX: ffffffffffffffda RBX: 00007fd840894000 RCX: 00007fd9a4f1283f
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592241] RDX: 000000000001c000 RSI: 00007fd840894000 RDI: 00000000000000e9
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592242] RBP: 00007fd9523f3520 R08: 0000000000000000 R09: 00007fd9523f355c
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592243] R10: 0000000000000000 R11: 0000000000000293 R12: 000000000001c000
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592244] R13: 0000000000000000 R14: 00000000044671b8 R15: 00007fd9819509a0
Oct 13 08:13:39 drago-server-ubuntu kernel: [ 9029.592247]  </TASK>
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320390] ------------[ cut here ]------------
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320394] WARNING: CPU: 0 PID: 153507 at mm/gup.c:142 try_grab_folio+0x84/0xd0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320400] Modules linked in: vhost_net vhost vhost_iotlb macvtap macvlan tap nf_conntrack_netlink nfnetlink_acct udp_diag tcp_diag inet_diag xt_recent wireguard curve25519_x86_64 libchacha20poly1305 chacha_x86_64 poly1305_x86_64 libcurve25519_generic libchacha ip6_udp_tunnel udp_tunnel xt_CHECKSUM xt_MASQUERADE nft_chain_nat nf_nat bridge stp llc zram ip6t_REJECT nf_reject_ipv6 blocklayoutdriver xt_hl ip6t_rt ipt_REJECT nf_reject_ipv4 xt_LOG nf_log_syslog xt_comment xt_multiport nft_limit xt_limit xt_addrtype nfsv4 xt_tcpudp xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nfs nft_compat netfs nf_tables nfnetlink binfmt_misc intel_rapl_msr intel_rapl_common mei_pxp mei_hdcp x86_pkg_temp_thermal intel_powerclamp kvm_intel kvm crct10dif_pclmul polyval_clmulni polyval_generic ghash_clmulni_intel sha256_ssse3 sha1_ssse3 aesni_intel crypto_simd cryptd rapl intel_cstate snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_codec_generic snd_hda_scodec_component snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320443]  nls_iso8859_1 snd_hda_codec snd_hda_core snd_hwdep zfs(POE) snd_pcm snd_seq_midi snd_seq_midi_event snd_rawmidi snd_seq spl(OE) snd_seq_device input_leds snd_timer joydev snd at24 soundcore mei_me mei intel_smartconnect mac_hid sch_fq_codel nct6775 nct6775_core hwmon_vid nfsd coretemp msr auth_rpcgss parport_pc nfs_acl lockd ppdev grace lp parport efi_pstore sunrpc ip_tables x_tables autofs4 raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx raid1 raid0 lz4 lz4_compress hid_generic usbhid hid i915 drm_buddy i2c_algo_bit ttm drm_display_helper cec rc_core i2c_i801 i2c_mux drm_kms_helper crc32_pclmul i2c_smbus ahci r8169 libahci drm realtek xhci_pci lpc_ich xhci_pci_renesas video wmi
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320484] CPU: 0 PID: 153507 Comm: connection Tainted: P    B      OE      6.10.14-ful01 #1
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320486] Hardware name: To Be Filled By O.E.M. To Be Filled By O.E.M./Z77 Pro3, BIOS P2.10 07/12/2013
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320487] RIP: 0010:try_grab_folio+0x84/0xd0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320489] Code: 00 48 63 d6 be 23 00 00 00 48 c1 e8 36 48 8b 3c c5 a0 39 7a 92 e8 6c 65 fe ff 31 c0 5d 31 d2 31 c9 31 f6 31 ff c3 cc cc cc cc <0f> 0b b9 f4 ff ff ff 89 c8 31 d2 31 c9 31 f6 31 ff c3 cc cc cc cc
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320491] RSP: 0018:ffffc0be0e9bf5a0 EFLAGS: 00010282
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320492] RAX: 0000000000000000 RBX: 0000000000210002 RCX: 00000000ffffffe5
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320493] RDX: 0000000000210002 RSI: 0000000000000001 RDI: fffff55b1f69b340
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320494] RBP: ffffc0be0e9bf5f8 R08: 0000000000000000 R09: 0000000000000000
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320495] R10: 0000000000000000 R11: 0000000000000000 R12: ffff9b074e1dff18
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320496] R13: 00007fd7d410c000 R14: ffff9b0886efe860 R15: 80000007da6cd225
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320497] FS:  00007fd9841fc640(0000) GS:ffff9b097f600000(0000) knlGS:0000000000000000
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320498] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320500] CR2: 00007fd7d410bffc CR3: 0000000259644005 CR4: 00000000001706f0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320501] Call Trace:
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320502]  <TASK>
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320505]  ? show_regs+0x6c/0x80
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320509]  ? __warn+0x88/0x140
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320511]  ? try_grab_folio+0x84/0xd0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320513]  ? report_bug+0x182/0x1b0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320517]  ? handle_bug+0x51/0xa0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320520]  ? exc_invalid_op+0x18/0x80
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320522]  ? asm_exc_invalid_op+0x1b/0x20
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320527]  ? try_grab_folio+0x84/0xd0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320529]  ? follow_page_pte+0x10d/0x700
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320531]  follow_page_mask+0x294/0xed0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320532]  ? handle_mm_fault+0xaf/0x330
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320534]  __get_user_pages+0x104/0x780
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320537]  __gup_longterm_locked+0xdb/0xe60
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320547]  gup_fast_fallback+0x146/0x1300
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320551]  get_user_pages_fast+0x33/0x80
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320552]  __iov_iter_get_pages_alloc+0xf2/0x610
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320557]  ? spl_kvmalloc+0x7a/0xb0 [spl]
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320567]  ? spl_kvmalloc+0x7a/0xb0 [spl]
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320575]  iov_iter_get_pages2+0x1e/0x50
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320577]  zfs_uio_get_dio_pages_alloc+0xe0/0x530 [zfs]
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320761]  zfs_setup_direct+0xda/0x180 [zfs]
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.320921]  zfs_write+0x1d3/0xe70 [zfs]
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321083]  ? vma_set_page_prot+0x69/0xc0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321087]  ? mprotect_fixup+0x1fa/0x390
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321090]  ? aa_file_perm+0x13d/0x540
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321093]  zpl_iter_write+0x119/0x1b0 [zfs]
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321238]  vfs_write+0x262/0x450
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321242]  __x64_sys_pwrite64+0xa6/0xd0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321244]  x64_sys_call+0x1e45/0x25c0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321246]  do_syscall_64+0x70/0x130
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321248]  ? __handle_mm_fault+0xcb5/0x1100
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321251]  ? __count_memcg_events+0x76/0x120
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321254]  ? count_memcg_events.constprop.0+0x2a/0x50
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321257]  ? handle_mm_fault+0xaf/0x330
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321258]  ? do_user_addr_fault+0x340/0x6c0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321261]  ? irqentry_exit_to_user_mode+0x56/0x180
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321264]  ? irqentry_exit+0x43/0x50
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321266]  ? exc_page_fault+0x93/0x1b0
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321268]  entry_SYSCALL_64_after_hwframe+0x76/0x7e
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321270] RIP: 0033:0x7fd9a4f1283f
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321291] Code: 08 89 3c 24 48 89 4c 24 18 e8 cd e1 f7 ff 4c 8b 54 24 18 48 8b 54 24 10 41 89 c0 48 8b 74 24 08 8b 3c 24 b8 12 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 31 44 89 c7 48 89 04 24 e8 0d e2 f7 ff 48 8b
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321292] RSP: 002b:00007fd9841ee1f0 EFLAGS: 00000293 ORIG_RAX: 0000000000000012
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321298] RAX: ffffffffffffffda RBX: 00007fd7d410c000 RCX: 00007fd9a4f1283f
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321299] RDX: 000000000001c000 RSI: 00007fd7d410c000 RDI: 00000000000004b7
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321300] RBP: 00007fd9841f0520 R08: 0000000000000000 R09: 00007fd9841f055c
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321301] R10: 0000000000000000 R11: 0000000000000293 R12: 000000000001c000
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321302] R13: 0000000000000000 R14: 00000000044671b8 R15: 00007fd8ed619510
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321304]  </TASK>
Oct 13 08:14:36 drago-server-ubuntu kernel: [ 9086.321305] ---[ end trace 0000000000000000 ]---

### mysql error.log

2024-10-13T05:48:01.648249+02:00 0 [System] [MY-015015] [Server] MySQL Server - start.
2024-10-13T05:48:01.922208+02:00 0 [Warning] [MY-010101] [Server] Insecure configuration for --secure-file-priv: Location is accessible to all OS users. Consider choosing a different directory.
2024-10-13T05:48:01.922415+02:00 0 [System] [MY-010116] [Server] /opt/mysql-8.4.2/bin/mysqld (mysqld 8.4.2) starting as process 6495
2024-10-13T05:48:02.051877+02:00 1 [System] [MY-013576] [InnoDB] InnoDB initialization has started.
2024-10-13T06:00:06.229023+02:00 1 [System] [MY-013577] [InnoDB] InnoDB initialization has ended.
2024-10-13T06:00:13.081187+02:00 0 [Warning] [MY-010068] [Server] CA certificate ca.pem is self signed.
2024-10-13T06:00:13.081264+02:00 0 [System] [MY-013602] [Server] Channel mysql_main configured to support TLS. Encrypted connections are now supported for this channel.
2024-10-13T06:00:13.090351+02:00 0 [Warning] [MY-011810] [Server] Insecure configuration for --pid-file: Location '/var/run/mysqld' in the path is accessible to all OS users. Consider choosing a different directory.
2024-10-13T06:00:13.391879+02:00 0 [System] [MY-011323] [Server] X Plugin ready for connections. Bind-address: '::' port: 33060, socket: /tmp/mysqlx.sock
2024-10-13T06:00:13.391974+02:00 0 [System] [MY-010931] [Server] /opt/mysql-8.4.2/bin/mysqld: ready for connections. Version: '8.4.2'  socket: '/var/run/mysqld/mysqld.sock'  port: 3306  MySQL Community Server - GPL.
2024-10-13T08:14:36.178130+02:00 16253 [Warning] [MY-012638] [InnoDB] Retry attempts for writing partial data failed.
2024-10-13T08:14:36.225187+02:00 16253 [ERROR] [MY-012639] [InnoDB] Write to file ./mysql_test_BMOTLWLI/#sql-ib17083-2763134737.ibd failed at offset 0, 114688 bytes should have been written, only 0 were written. Operating system error number 12. Check that your OS and file system support files of this size. Check also that the disk is not full or a disk quota exceeded.
2024-10-13T08:14:36.225245+02:00 16253 [ERROR] [MY-012640] [InnoDB] Error number 12 means 'Cannot allocate memory'
2024-10-13T08:14:36.225758+02:00 16253 [ERROR] [MY-012129] [InnoDB] Could not write the first page to tablespace './mysql_test_BMOTLWLI/#sql-ib17083-2763134737.ibd'
2024-10-13T08:14:40.359720+02:00 16254 [Warning] [MY-012638] [InnoDB] Retry attempts for writing partial data failed.
2024-10-13T08:14:40.359778+02:00 16254 [ERROR] [MY-012129] [InnoDB] Could not write the first page to tablespace './mysql_test_LRNFJGRH/#sql-ib17129-2763134803.ibd'
2024-10-13T08:20:07.393529+02:00 16957 [System] [MY-013172] [Server] Received SHUTDOWN from user drago. Shutting down mysqld (Version: 8.4.2).
2024-10-13T08:20:09.434790+02:00 0 [Warning] [MY-010909] [Server] /opt/mysql-8.4.2/bin/mysqld: Forcing close of thread 1158  user: 'drago'.
2024-10-13T08:20:11.359273+02:00 0 [System] [MY-010910] [Server] /opt/mysql-8.4.2/bin/mysqld: Shutdown complete (mysqld 8.4.2)  MySQL Community Server - GPL.
2024-10-13T08:20:11.359312+02:00 0 [System] [MY-015016] [Server] MySQL Server - end.

-->

robn commented 1 day ago

Thanks for the report, loads of great detail there.

I don't know how many MySQL admins we have here, but I used to run a couple of instances long ago, so I'm gonna try and reproduce this. Could you help by describing the "intensive test" in more detail? Alternatively, do you know if there's any convenient load generator or similar for MySQL that might trip it? Also, your full MySQL config would be useful - I remember there's some other tuning often needed to make it work well on ZFS, and I wouldn't want anything to mask the issue.

Thanks!

robn commented 1 day ago

@bwatkinson anything in that backtrace look obvious to you?

bwatkinson commented 21 hours ago

@bwatkinson anything in that backtrace look obvious to you?

Nothing immediately obvious to me stands out looking at the backtrace from the kernel. It is odd that we are getting a bad page state when freeing up the pages.

drago72 commented 1 hour ago

Sorry for the delayed response.

I've conducted further investigations and observed that when MySQL uses direct I/O (innodb_flush_method = O_DIRECT) on a dataset with sync = standard (changed from none), the errors reported by MySQL change from:

(sync = None)
2024-10-13T08:14:36.225187+02:00 16253 [ERROR] [MY-012639] [InnoDB] Write to file ./mysql_test_BMOTLWLI/#sql-ib17083-2763134737.ibd failed at offset 0, 114688 bytes should have been written, only 0 were written. Operating system error number 12. Check that your OS and file system support files of this size. Check also that the disk is not full or a disk quota exceeded.
2024-10-13T08:14:36.225245+02:00 16253 [ERROR] [MY-012640] [InnoDB] Error number 12 means 'Cannot allocate memory'
2024-10-13T08:14:36.225758+02:00 16253 [ERROR] [MY-012129] [InnoDB] Could not write the first page to tablespace './mysql_test_BMOTLWLI/#sql-ib17083-2763134737.ibd'

to:

(sync = standard)
2024-10-16T09:32:28.139769+02:00 105 [ERROR] [MY-012639] [InnoDB] Write to file ./mysql_test_PWZFDTWD/table_1.ibd failed at offset 0, 114688 bytes should have been written, only 0 were written. Operating system error number 12. Check that your OS and file system support files of this size. Check also that the disk is not full or a disk quota exceeded.
2024-10-16T09:32:28.139805+02:00 105 [ERROR] [MY-012640] [InnoDB] Error number 12 means 'Cannot allocate memory'
2024-10-16T09:32:28.139846+02:00 105 [ERROR] [MY-012129] [InnoDB] Could not write the first page to tablespace './mysql_test_PWZFDTWD/table_1.ibd'
2024-10-16T09:32:28.141953+02:00 105 [ERROR] [MY-013133] [Server] Create table/tablespace 'table_1' failed, as disk is full.

This might help in understanding the issue.

I've quickly rewritten the "intensive test" script to make it public. This script reproduces the problem.

#!/usr/bin/env python3.12

from typing import Any
from random import choice as rChoice, randint as rRandint, seed as rSeed, sample as rSample
from datetime import datetime
from time import time as tTime, sleep as tSleep
from string import ascii_uppercase
from json import loads as jLoads
from numpy import mean as npMean

from threading import Thread

# python3.12 -m pip install -U mysql-connector-python
from mysql.connector import Error as mSQLError, Warning as mSQLWarning, __version_info__ as mSQLversion
from mysql.connector import MySQLConnection, CMySQLConnection, HAVE_CEXT

class statistiche():
    def __init__(self):
        self.tempi = {}
        self.tCreate = 0.0
        self.tInsert = 0.0
        self.tOpt = 0.0
        self.tSelect = 0.0
        self.tUpdate = 0.0
        self.tInfo = 0.0
        self.tTot = 0.0

class benchmark(Thread, statistiche):
    __cfgConn: dict = {'user': 'root',
                       'passwd': 'password_root',
                       'unix_socket': '/var/run/mysqld/mysqld.sock',
                       'connect_timeout': 600,
                       'autocommit': True,
                       'charset': 'utf8mb4'}

    def __init__(self, tId, nrRip=3, nrTab=50, nrRowTab=100, nrSelUpd=4000) -> None:
        Thread.__init__(self)
        statistiche.__init__(self)

        self.threadID = tId
        self.ritorno = ""
        self.__colConvJson = []

        self.__nomeSk = "mysql_test_" + "".join(rChoice(ascii_uppercase) for _ in range(8))
        self.__nrRip = nrRip
        self.__nrTab = nrTab
        self.__nrRowTab = nrRowTab
        self.__nrSelUpd = nrSelUpd

        self.__text_rnd = []
        for _ in range(100):
            self.__text_rnd.append("".join(rChoice(ascii_uppercase) for _ in range(16)))

    def __byteToString(self, valore: Any, tipo: str = "utf-8", corr: str = None, returnErr: str = None) -> str:
        try:
            if isinstance(valore, bytes):
                if corr is None:
                    rit = valore.decode(tipo)
                else:
                    rit = valore.decode(tipo, errors=corr)
            else:
                rit = str(valore)
            return rit.strip()
        except Exception:
            return returnErr

    def __convertiTipo(self, valore: Any, tipo: str) -> Any:
        if valore is None:
            return None
        if tipo in ('tinyint', 'mediumint', 'int', 'long'):
            try:
                return int(valore)
            except Exception:
                return 0
        elif tipo in ('float', 'double'):
            try:
                return float(valore)
            except Exception:
                return 0.0
        elif tipo == 'date':
            valore = self.__byteToString(valore, returnErr='')
            try:
                return datetime.strptime(valore[0:10], '%Y-%m-%d').date()
            except Exception:
                return None
        elif tipo == 'datetime':
            valore = self.__byteToString(valore, returnErr='')
            try:
                if len(valore) == 10:
                    return datetime.strptime(valore, '%Y-%m-%d 00:00:00')
                else:
                    return datetime.strptime(valore, '%Y-%m-%d %H:%M:%S')
            except Exception:
                return None
        elif tipo == 'Decimal':
            try:
                return float(valore)
            except Exception:
                return 0.0
        else:
            return str(valore)

    def __convertRit(self, ritorno: Any, ritTipo: str) -> Any:
        if ritorno in (None, ""):
            return None
        if isinstance(ritorno, tuple) and len(ritorno) == 0:
            return None
        elif ritTipo in ('int', 'long', 'str', 'float', 'double', 'date', 'datetime'):
            return self.__convertiTipo(ritorno[0][0], ritTipo)
        elif ritTipo in ('list', 'list1'):
            rit = list(ritorno)
            for i in range(rit.__len__()):
                rit[i] = list(rit[i])
                if self.__descriptRit:
                    for c in range(rit[i].__len__()):
                        if self.__descriptRit[c][0] in self.__colConvJson:
                            rit[i][c] = jLoads(self.__byteToString(rit[i][c], corr="ignore"))
                if ritTipo == 'list1':
                    break
            if ritTipo == 'list1' and len(rit) > 0:
                rit = rit[0]
            return rit
        elif ritTipo in ('dict', 'dict1'):
            rit = list(ritorno)
            for row in rit:
                if self.__descriptRit:
                    for col in self.__colConvJson:
                        if col in list(row.keys()):
                            row[col] = jLoads(self.__byteToString(row[col], corr="ignore"))  # cls=json_encoder
                if ritTipo == 'dict1':
                    break
            if ritTipo == 'dict1' and len(rit) > 0:
                rit = rit[0]
            return rit
        elif ritTipo == 'tuple1':
            rit = tuple(ritorno)
            if len(rit) > 0:
                rit = rit[0]
            return rit
        else:
            return ritorno

    def connetti(self) -> None:
        try:
            if mSQLversion > (2, 1) and HAVE_CEXT:
                self.__cfgConn["use_pure"] = False
                self.__conn = CMySQLConnection(**self.__cfgConn)
            else:
                self.__conn = MySQLConnection(**self.__cfgConn)
            self.__conn.get_warnings = False
        except Exception:
            ...

    def chiudi(self) -> None:
        if self.__conn and hasattr(self.__conn, "close") and callable(getattr(self.__conn, "close")):
            self.__conn.close()

    def query(self, ritTipo: str, query: str) -> Any:
        rit = None
        self.query_err = ""
        self.__descriptRit = None
        if ritTipo[:4] == "dict":
            cur = self.__conn.cursor(buffered=True, dictionary=True)
        else:
            cur = self.__conn.cursor(buffered=True)
        try:
            cur.execute(query)
            nrRow: int = cur.rowcount if cur.with_rows else 0
            if nrRow > 0:
                if ritTipo != "":
                    rit = cur.fetchall()
                if len(self.__colConvJson) > 0:
                    self.__descriptRit = cur.description
        except (mSQLError, mSQLWarning) as err:
            self.query_err = err.msg if hasattr(err, "msg") else ""
            rit = None
        finally:
            if cur._executed:
                cur.close()
        if rit is not None:
            rit = self.__convertRit(rit, ritTipo)
        return rit

    def __name_tab(self, i) -> str:
        return "table_" + str(i)

    def __name_tab_compl(self, i) -> str:
        return self.__nomeSk + "." + self.__name_tab(i)

    def __tab_rnd(self) -> str:
        return self.__name_tab(rRandint(0, self.__nrTab - 1))

    def __tab_rnd_compl(self) -> str:
        return self.__nomeSk + "." + self.__tab_rnd()

    def run(self) -> None:
        txt_L: int = len(self.__text_rnd) - 1
        lista_id: list[int] = [i for i in range(1, self.__nrRowTab + 1)]

        listaCrk = []
        listaIns = []
        listaUpd = []
        listaSel = []
        listaInf = []
        listaTot = []

        self.connetti()

        for _ in range(self.__nrRip):
            # creo database di test
            if self.query('int', "SELECT COUNT(*) FROM INFORMATION_SCHEMA.SCHEMATA WHERE "
                          "SCHEMA_NAME = '" + self.__nomeSk + "'") > 0:
                self.query('', "DROP DATABASE " + self.__nomeSk)
            self.query('', "CREATE DATABASE " + self.__nomeSk)

            tSleep(3.3 * self.threadID)

            oraIniT = oraIni = tTime()
            # creo 100 tabelle sia MyISAM che InnoDB
            for i in range(self.__nrTab):
                if i % 2 == 0:
                    tipoEn: str = "MyISAM"
                else:
                    tipoEn: str = "InnoDB"
                self.query('', "CREATE TABLE " + self.__name_tab_compl(i) + " ("
                               "id INT UNSIGNED NOT NULL AUTO_INCREMENT,"
                               "valInt INT NULL,"
                               "valChar VARCHAR(16000) NULL,"
                               "valJSON JSON NULL,"
                               "PRIMARY KEY (id)) ENGINE = " + tipoEn)
            self.tCreate = tTime() - oraIni

            # inserisco dati sia INSERT che REPLACE
            doc_json: dict = '{"pippo": 2, "pluto": 3.5, "paperino": [1,2,3]}'
            oraIni: float = tTime()
            for i in range(self.__nrTab):
                nomeTab: str = self.__name_tab_compl(i)
                for v in lista_id:
                    testo: str = self.__text_rnd[rRandint(0, txt_L)]
                    if i % 2 == 0:
                        self.query('', f"INSERT INTO {nomeTab} (valInt, valChar, valJSON) VALUES "
                                   f"({v}, '{testo}', '{doc_json}')")
                    else:
                        self.query('', f"REPLACE {nomeTab}  (id, valInt, valChar, valJSON) VALUES "
                                   f"({v}, {v}, '{testo}', '{doc_json}')")
            self.tInsert = tTime() - oraIni

            # select sulle tabelle
            oraIni = tTime()
            for _ in range(self.__nrSelUpd):
                v: int = rRandint(1, self.__nrRowTab)
                nomeTab = self.__tab_rnd_compl()
                rit_int = self.query('int', f"SELECT valInt FROM {nomeTab} WHERE id = {v}")
                assert isinstance(rit_int, int)
                rit_str = self.query('str', f"SELECT valChar FROM {nomeTab} WHERE valInt = {v}")
                assert isinstance(rit_str, str)
                rit_dict = self.query('dict1', f"SELECT valJSON FROM {nomeTab} WHERE id ={v}")
                assert isinstance(rit_dict, dict)
                rit_dict = jLoads(rit_dict["valJSON"])
                assert isinstance(rit_dict, dict)
                ids: str = "(" + ",".join([str(id) for id in rSample(lista_id, 5)]) + ")"
                rit_list = self.query('list', f"SELECT * FROM {nomeTab} WHERE id IN {ids}")
                assert isinstance(rit_list, list)
            self.tSelect = tTime() - oraIni

            # aggiorno dati
            oraIni = tTime()
            for _ in range(self.__nrSelUpd):
                v: int = rRandint(1, self.__nrRowTab)
                self.query('', f"UPDATE {self.__tab_rnd_compl()} SET valChar = '' WHERE id = {v}")
            self.tUpdate = tTime() - oraIni

            # select su information_schema
            oraIni = tTime()
            for _ in range(self.__nrSelUpd):
                nomeTab: str = self.__tab_rnd()
                nT: int = rRandint(1, 3)
                if nT == 1:
                    rit_str = self.query('str', "SELECT ENGINE FROM information_schema.TABLES WHERE "
                                         f"TABLE_SCHEMA = '{self.__nomeSk}' AND TABLE_NAME = '{nomeTab}'")
                    assert isinstance(rit_str, str)
                elif nT == 2:
                    rit_int = self.query('int', "SELECT TABLE_ROWS FROM information_schema.TABLES WHERE "
                                         f"TABLE_SCHEMA = '{self.__nomeSk}' AND TABLE_NAME = '{nomeTab}'")
                    assert isinstance(rit_int, int)
                else:
                    rit_datetime = self.query('datetime', "SELECT UPDATE_TIME FROM information_schema.TABLES WHERE "
                                              f"TABLE_SCHEMA = '{self.__nomeSk}' AND TABLE_NAME = '{nomeTab}'")
                    assert isinstance(rit_datetime, datetime | None)
            self.tInfo = tTime() - oraIni

            # cancello tutto il database di test
            self.query('', f"DROP DATABASE {self.__nomeSk}")

            self.tTot = tTime() - oraIniT

            listaCrk.append(self.tCreate)
            listaIns.append(self.tInsert)
            listaUpd.append(self.tUpdate)
            listaSel.append(self.tSelect)
            listaInf.append(self.tInfo)
            listaTot.append(self.tTot)

        self.chiudi()

        self.tempi['crk'] = round(npMean(listaCrk), 3)
        self.tempi['ins'] = round(npMean(listaIns), 3)
        self.tempi['upd'] = round(npMean(listaUpd), 3)
        self.tempi['sel'] = round(npMean(listaSel), 3)
        self.tempi['inf'] = round(npMean(listaInf), 3)
        self.tempi['tot'] = round(npMean(listaTot), 3)

        self.ritorno = f"Tempi [{self.threadID}]: {self.tempi}"

def main() -> None:
    nrRip: int = 3
    nrTab: int = 50
    nrRowTab: int = 100
    nrSelUpd: int = 4000
    nrTr: int = 8
    print(f"Configurazione: nrRip={nrRip}, nrTab={nrTab}, nrRowTab={nrRowTab}, nrSelUpd={nrSelUpd}")

    print("")
    print(f"Avvio sessioni di test a {nrTr} thread ...")
    print(f"Ora avvio: {datetime.now()}")
    threads = []
    for tId in range(nrTr):
        t: benchmark = benchmark(tId, nrRip=nrRip, nrTab=nrTab, nrRowTab=nrRowTab, nrSelUpd=nrSelUpd)
        threads.append(t)
    for t in threads:
        t.start()
    for t in threads:
        t.join()
        print(t.ritorno)
    tempo: float = 0.0
    for t in threads:
        tempo += t.tempi['tot']
    print(f"Ora fine: {datetime.now()}")

    print(f"Terminate sessioni di test a {nrTr} thread. Somma dei tempi: {round(tempo, 2)} sec.")

if __name__ == '__main__':
    rSeed()

    main()

MySQL configuration files are attached below.

/etc/mysql/mysql.conf.d/mysqld.cnf

[mysqld]
pid-file                    = /var/run/mysqld/mysqld.pid
socket              = /var/run/mysqld/mysqld.sock
#datadir            = /zfs1/database_8.4
datadir             = /zfs1/database_8.4_copy
log-error           = /opt/mysql-8.4.2/log/error.log

# default = /usr/
basedir             = /opt/mysql-8.4.2/

# default = /usr/share/mysql-8.0/
lc_messages_dir     = /opt/mysql-8.4.2/share/

# default = /usr/share/mysql-8.0/charsets/
character_sets_dir  = /opt/mysql-8.4.2/share/charsets/

# default = /usr/lib/mysql/plugin/
plugin_dir          = /opt/mysql-8.4.2/lib/plugin/

/etc/mysql/mysql.conf.d/drago_mod.cnf

[mysqld]
# in caso di emergenza
# https://dev.mysql.com/doc/refman/8.0/en/forcing-innodb-recovery.html
# da 1 a 6 (ma 4,5,6 sconsigliati)
#innodb_force_recovery = 6

# non server perche' non uso la replicazione
skip-log-bin

log_timestamps                  = SYSTEM
secure-file-priv                    = /var/lib/mysql-files/

# default = 8161
# [Warning] [MY-010140] [Server] Could not increase number of max_open_files to more than 10000 (request: 30000)
#open_files_limit                   = 30000

# default = 30
net_read_timeout                = 60

# default = ON
event-scheduler                 = ON

# default = InnoDB
default_storage_engine          = MyISAM
# default = InnoDB
#default_tmp_storage_engine     = MyISAM

# default = 151
max_connections                 = 101

# log e query lente per capire dove ottimizzare
log_output                      = TABLE
general_log                     = OFF
slow_query_log                  = ON
long_query_time                 = 15

# grandezza massima tabelle MEMORY
max_heap_table_size             = 512M

### MyISAM variables
# default = OFF
myisam_use_mmap             = ON
key_buffer_size                 = 128M
myisam-block-size               = 16384

### InnoDB variables
# default = 134217728
innodb_buffer_pool_size         = 1G
# default = fsync, consigliato O_DIRECT
#innodb_flush_method            = fsync
innodb_flush_method             = O_DIRECT
# default = ON (dovrebbe evitare la scansione di tutti tablespace) -> OFF
innodb-validate-tablespace-paths= OFF
# default = 1
innodb_flush_log_at_trx_commit  = 2
# default = ON possibile ON, OFF, DETECT_AND_RECOVER (uguale ON), DETECT_ONLY (solo metadata)
innodb_doublewrite              = OFF
# default = crc32
innodb_checksum_algorithm       = NONE
# default = ON
innodb_log_checksums            = OFF
# default = 1
innodb_page_cleaners            = 4
# default = 1
innodb_buffer_pool_instances    = 4

# default = 
innodb_tmpdir                   = /var/tmp/
# default = ibtmp1:12M:autoextend
innodb_temp_data_file_path      = /var/tmp/ib_tmp1:32M;/var/tmp/ib_tmp2:32M:autoextend
# default = ./#innodb_temp/
innodb_temp_tablespaces_dir     = /var/tmp/

Thanks for your interest and effort on this.