openzfs / zfs

OpenZFS on Linux and FreeBSD
https://openzfs.github.io/openzfs-docs
Other
10.42k stars 1.72k forks source link

system freezes when starting to swap to zvol #1798

Closed maci0 closed 8 years ago

maci0 commented 10 years ago
root@nb0815 ~ # free -m
             total       used       free     shared    buffers     cached
Mem:          3832       1735       2097          0          0        230
-/+ buffers/cache:       1504       2328
Swap:         8191          0       8191

root@nb0815 ~ # zpool status
  pool: rpool
 state: ONLINE
  scan: scrub repaired 0 in 0h31m with 0 errors on Thu Oct 17 18:55:01 2013
config:

    NAME                              STATE     READ WRITE CKSUM
    rpool                             ONLINE       0     0     0
      ata-ST9250410AS_5VGB002J-part2  ONLINE       0     0     0

errors: No known data errors

130 root@nb0815 ~ # zfs get all rpool/swap                                   :(
NAME        PROPERTY               VALUE                  SOURCE
rpool/swap  type                   volume                 -
rpool/swap  creation               Wed Oct  2 23:53 2013  -
rpool/swap  used                   8.50G                  -
rpool/swap  available              152G                   -
rpool/swap  referenced             264M                   -
rpool/swap  compressratio          2.59x                  -
rpool/swap  reservation            none                   default
rpool/swap  volsize                8G                     local
rpool/swap  volblocksize           4K                     -
rpool/swap  checksum               on                     default
rpool/swap  compression            on                     local
rpool/swap  readonly               off                    default
rpool/swap  copies                 1                      default
rpool/swap  refreservation         8.50G                  local
rpool/swap  primarycache           metadata               local
rpool/swap  secondarycache         none                   local
rpool/swap  usedbysnapshots        0                      -
rpool/swap  usedbydataset          264M                   -
rpool/swap  usedbychildren         0                      -
rpool/swap  usedbyrefreservation   8.25G                  -
rpool/swap  logbias                latency                default
rpool/swap  dedup                  off                    default
rpool/swap  mlslabel               none                   default
rpool/swap  sync                   always                 local
rpool/swap  refcompressratio       2.59x                  -
rpool/swap  written                264M                   -
rpool/swap  snapdev                hidden                 default
rpool/swap  com.sun:auto-snapshot  false                  local

root@nb0815 ~ # swapon -s      
Filename                Type        Size    Used    Priority
/dev/zd0                                partition   8388604 0   -1

root@nb0815 ~ # zfs list -t all
NAME                                                  USED  AVAIL  REFER  MOUNTPOINT
rpool                                                84.7G   144G    31K  none
rpool/ROOT                                           6.64G   144G  4.27G  /
rpool/ROOT@zfs-auto-snap_daily-2013-10-05-1808       16.0M      -  2.22G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-06-0114       14.4M      -  2.21G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-07-0944       21.1M      -  2.27G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-08-0111       17.6M      -  2.28G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-09-1431       8.09M      -  2.21G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-10-0935       1.33M      -  2.21G  -
rpool/ROOT@zfs-auto-snap_weekly-2013-10-10-0955      1.34M      -  2.21G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-11-0834       12.1M      -  2.24G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-12-0144       11.6M      -  2.30G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-13-0123       13.4M      -  2.30G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-14-0920       15.7M      -  3.09G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-15-1019       18.6M      -  3.11G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-16-0143       84.8M      -  3.21G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-17-1125       33.1M      -  3.23G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-18-0127        680K      -  3.50G  -
rpool/ROOT@zfs-auto-snap_weekly-2013-10-18-0147       684K      -  3.50G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-19-1150       16.0M      -  3.51G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-20-0126        949K      -  3.56G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-0501       980K      -  3.56G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-1001      5.54M      -  3.56G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-1101      2.69M      -  3.56G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-1201      2.08M      -  3.56G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-1501      3.15M      -  3.56G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-1601      2.33M      -  3.56G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-1701      3.73M      -  3.56G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-1801      7.25M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-1901      7.09M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-2101      1.32M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-2201      1.34M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-20-2301      1.23M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-0001      1.23M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-0101      1.23M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_daily-2013-10-21-0138       1.21M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-0201      1.22M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-0301      1.20M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-0401      1.22M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-0501      1.25M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-0601      1.27M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-0701      3.46M      -  3.57G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-1001      15.2M      -  3.62G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-1101      16.5M      -  4.27G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-1201       826K      -  4.27G  -
rpool/ROOT@zfs-auto-snap_frequent-2013-10-21-1215     785K      -  4.27G  -
rpool/ROOT@zfs-auto-snap_frequent-2013-10-21-1230     812K      -  4.27G  -
rpool/ROOT@zfs-auto-snap_frequent-2013-10-21-1245     890K      -  4.27G  -
rpool/ROOT@zfs-auto-snap_frequent-2013-10-21-1300     808K      -  4.27G  -
rpool/ROOT@zfs-auto-snap_hourly-2013-10-21-1301      1.07M      -  4.27G  -
rpool/backup                                         58.5M   144G  58.1M  /backup
rpool/backup@zfs-auto-snap_daily-2013-10-05-1808         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-06-0114         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-07-0944         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-08-0111         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-09-1431         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-10-0935         0      -  58.1M  -
rpool/backup@zfs-auto-snap_weekly-2013-10-10-0955        0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-11-0834         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-12-0144         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-13-0123         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-14-0920         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-15-1019         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-16-0143         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-17-1125         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-18-0127         0      -  58.1M  -
rpool/backup@zfs-auto-snap_weekly-2013-10-18-0147        0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-19-1150         0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-20-0126         0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-0501        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-1001        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-1101        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-1201        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-1501        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-1601        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-1701        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-1801        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-1901        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-2101        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-2201        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-20-2301        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-0001        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-0101        0      -  58.1M  -
rpool/backup@zfs-auto-snap_daily-2013-10-21-0138         0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-0201        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-0301        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-0401        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-0501        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-0601        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-0701        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-1001        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-1101        0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-1201        0      -  58.1M  -
rpool/backup@zfs-auto-snap_frequent-2013-10-21-1215      0      -  58.1M  -
rpool/backup@zfs-auto-snap_frequent-2013-10-21-1230      0      -  58.1M  -
rpool/backup@zfs-auto-snap_frequent-2013-10-21-1245      0      -  58.1M  -
rpool/backup@zfs-auto-snap_frequent-2013-10-21-1300      0      -  58.1M  -
rpool/backup@zfs-auto-snap_hourly-2013-10-21-1301        0      -  58.1M  -
rpool/downloads                                      18.5G   144G  18.5G  /home/maci/Downloads
rpool/home                                           28.4G   144G  23.7G  /home
rpool/home@zfs-auto-snap_daily-2013-10-05-1808       10.9M      -  12.8G  -
rpool/home@zfs-auto-snap_daily-2013-10-06-0114       41.9M      -  12.8G  -
rpool/home@zfs-auto-snap_daily-2013-10-07-0944        272M      -  12.1G  -
rpool/home@zfs-auto-snap_daily-2013-10-08-0111       27.1M      -  11.9G  -
rpool/home@zfs-auto-snap_daily-2013-10-09-1431       14.5M      -  11.9G  -
rpool/home@zfs-auto-snap_daily-2013-10-10-0935       57.5K      -  12.9G  -
rpool/home@zfs-auto-snap_weekly-2013-10-10-0955      57.5K      -  12.9G  -
rpool/home@zfs-auto-snap_daily-2013-10-11-0834       25.0M      -  12.9G  -
rpool/home@zfs-auto-snap_daily-2013-10-12-0144       64.3M      -  12.9G  -
rpool/home@zfs-auto-snap_daily-2013-10-13-0123       26.2M      -  20.8G  -
rpool/home@zfs-auto-snap_daily-2013-10-14-0920        303M      -  21.1G  -
rpool/home@zfs-auto-snap_daily-2013-10-15-1019       36.5M      -  20.8G  -
rpool/home@zfs-auto-snap_daily-2013-10-16-0143       14.6M      -  22.4G  -
rpool/home@zfs-auto-snap_daily-2013-10-17-1125       15.1M      -  23.4G  -
rpool/home@zfs-auto-snap_daily-2013-10-18-0127         74K      -  23.7G  -
rpool/home@zfs-auto-snap_weekly-2013-10-18-0147        77K      -  23.7G  -
rpool/home@zfs-auto-snap_daily-2013-10-19-1150       58.8M      -  23.7G  -
rpool/home@zfs-auto-snap_daily-2013-10-20-0126       6.93M      -  24.0G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-0501      6.91M      -  24.0G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-1001       164M      -  23.8G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-1101      9.86M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-1201      11.2M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-1501      9.40M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-1601      9.32M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-1701       193M      -  23.9G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-1801      8.59M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-1901      8.30M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-2101      6.94M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-2201      6.84M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-20-2301      6.83M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-0001      6.84M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-0101      4.17M      -  23.7G  -
rpool/home@zfs-auto-snap_daily-2013-10-21-0138        478K      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-0201      3.13M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-0301      6.83M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-0401      6.84M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-0501      6.83M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-0601      6.84M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-0701      7.33M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-1001      18.1M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-1101      8.36M      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-1201          0      -  23.7G  -
rpool/home@zfs-auto-snap_frequent-2013-10-21-1215        0      -  23.7G  -
rpool/home@zfs-auto-snap_frequent-2013-10-21-1230        0      -  23.7G  -
rpool/home@zfs-auto-snap_frequent-2013-10-21-1245    88.5K      -  23.7G  -
rpool/home@zfs-auto-snap_frequent-2013-10-21-1300      73K      -  23.7G  -
rpool/home@zfs-auto-snap_hourly-2013-10-21-1301       140K      -  23.7G  -
rpool/images                                         22.5G   144G  22.5G  /images
rpool/images@zfs-auto-snap_daily-2013-10-05-1808         0      -  22.0G  -
rpool/images@zfs-auto-snap_daily-2013-10-06-0114         0      -  22.0G  -
rpool/images@zfs-auto-snap_daily-2013-10-07-0944         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-08-0111         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-09-1431         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-10-0935         0      -  22.5G  -
rpool/images@zfs-auto-snap_weekly-2013-10-10-0955        0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-11-0834         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-12-0144         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-13-0123         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-14-0920         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-15-1019         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-16-0143         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-17-1125         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-18-0127         0      -  22.5G  -
rpool/images@zfs-auto-snap_weekly-2013-10-18-0147        0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-19-1150         0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-20-0126         0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-0501        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-1001        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-1101        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-1201        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-1501        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-1601        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-1701        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-1801        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-1901        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-2101        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-2201        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-20-2301        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-0001        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-0101        0      -  22.5G  -
rpool/images@zfs-auto-snap_daily-2013-10-21-0138         0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-0201        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-0301        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-0401        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-0501        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-0601        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-0701        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-1001        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-1101        0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-1201        0      -  22.5G  -
rpool/images@zfs-auto-snap_frequent-2013-10-21-1215      0      -  22.5G  -
rpool/images@zfs-auto-snap_frequent-2013-10-21-1230      0      -  22.5G  -
rpool/images@zfs-auto-snap_frequent-2013-10-21-1245      0      -  22.5G  -
rpool/images@zfs-auto-snap_frequent-2013-10-21-1300      0      -  22.5G  -
rpool/images@zfs-auto-snap_hourly-2013-10-21-1301        0      -  22.5G  -
rpool/root                                            991K   144G   104K  /root
rpool/root@zfs-auto-snap_daily-2013-10-05-1808         30K      -  68.5K  -
rpool/root@zfs-auto-snap_daily-2013-10-06-0114         31K      -  72.5K  -
rpool/root@zfs-auto-snap_daily-2013-10-07-0944         33K      -  74.5K  -
rpool/root@zfs-auto-snap_daily-2013-10-08-0111         30K      -    74K  -
rpool/root@zfs-auto-snap_daily-2013-10-09-1431         30K      -  75.5K  -
rpool/root@zfs-auto-snap_daily-2013-10-10-0935           0      -  74.5K  -
rpool/root@zfs-auto-snap_weekly-2013-10-10-0955          0      -  74.5K  -
rpool/root@zfs-auto-snap_daily-2013-10-11-0834       30.5K      -  75.5K  -
rpool/root@zfs-auto-snap_daily-2013-10-12-0144          1K      -    77K  -
rpool/root@zfs-auto-snap_daily-2013-10-13-0123          1K      -    77K  -
rpool/root@zfs-auto-snap_daily-2013-10-14-0920         35K      -    79K  -
rpool/root@zfs-auto-snap_daily-2013-10-15-1019         34K      -    78K  -
rpool/root@zfs-auto-snap_daily-2013-10-16-0143         53K      -  98.5K  -
rpool/root@zfs-auto-snap_daily-2013-10-17-1125       41.5K      -   104K  -
rpool/root@zfs-auto-snap_daily-2013-10-18-0127           0      -   106K  -
rpool/root@zfs-auto-snap_weekly-2013-10-18-0147          0      -   106K  -
rpool/root@zfs-auto-snap_daily-2013-10-19-1150         35K      -   104K  -
rpool/root@zfs-auto-snap_daily-2013-10-20-0126           0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-0501          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-1001        35K      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-1101          0      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-1201          0      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-1501          0      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-1601          0      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-1701          0      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-1801          0      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-1901          0      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-2101          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-2201          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-20-2301          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-0001          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-0101          0      -   106K  -
rpool/root@zfs-auto-snap_daily-2013-10-21-0138           0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-0201          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-0301          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-0401          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-0501          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-0601          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-0701          0      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-1001         1K      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-1101      36.5K      -   106K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-1201          0      -   104K  -
rpool/root@zfs-auto-snap_frequent-2013-10-21-1215        0      -   104K  -
rpool/root@zfs-auto-snap_frequent-2013-10-21-1230        0      -   104K  -
rpool/root@zfs-auto-snap_frequent-2013-10-21-1245        0      -   104K  -
rpool/root@zfs-auto-snap_frequent-2013-10-21-1300        0      -   104K  -
rpool/root@zfs-auto-snap_hourly-2013-10-21-1301      36.5K      -   104K  -
rpool/swap                                           8.50G   152G   264M  -
root@nb0815 ~ # 

as soon as I run

memtester 3000m
my system freezes. same happens when I just use it normally and some java process happens to eat up most of my memory

behlendorf commented 10 years ago

@maci0 What version of ZFS are you using? It would also be helpful to see the console logs, they should indicate which threads are blocked and why.

maci0 commented 10 years ago

@behlendorf im using version 0.6.2

no idea how to get the console log, journald doesnt log anymore when this happens. this whole system just freezes and never recovers

sysrq doesnt really work either

maci0 commented 10 years ago

@behlendorf I think this should be pretty easy to reproduce anyway.

behlendorf commented 10 years ago

@maci0 OK, we'll look it to it when we get a chance. Although this will probably be after some additional memory management work is completed.

ryao commented 10 years ago

@maci0 I am already working on a fix for this. The plan is to have it in 0.6.3.

maci0 commented 10 years ago

@ryao , awesome :)

for now I have limited the arc cache to prevent my system from freezing randomly. once 0.6.3 is released i will test again

hcoin commented 10 years ago

I confirm this bug on ubuntu saucy. Here are the steps to repeat it, and following that a syslog trace showing the problem. You can use this to test whether your fix is enough. This PPA contains the current stable 0.6.1 release.

To repeat: 1: Create a zpool with at least two partitions in a mirror, and zvol 1.5 times system memory. mkswap and set the system to use that as it's only swap space. Create another zpool with a mirror and another zvol, put a file system on it and run a script that changes the content all the time.

Create another script that does snapshots on the zvol with the changing files. Fill up system memory so the swap is engaged. Sooner or later you will get this core dump and a hang. There is no recovery.

... Nov 18 20:00:01 noc3 CRON[18437]: (root) CMD (zfs-auto-snapshot -q -g --label=frequent --keep=4 //) Nov 18 20:00:35 noc3 zfs-auto-snap: @zfs-auto-snap_frequent-2013-11-18-2000, 1 created, 1 destroyed, 0 warnings. Nov 18 20:11:24 noc3 kernel: [36240.776022] INFO: task kworker/2:0:35 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.776028] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.776031] kworker/2:0 D ffff88041fc94580 0 35 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.776064] Workqueue: events async_pf_execute [kvm] Nov 18 20:11:24 noc3 kernel: [36240.776066] ffff88040968db28 0000000000000046 ffff88040968dfd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.776071] ffff88040968dfd8 0000000000014580 ffff880409691770 ffff88041fc94e28 Nov 18 20:11:24 noc3 kernel: [36240.776080] ffff88040968dbb0 0000000000000002 ffffffff8113ead0 ffff88040968dba0 Nov 18 20:11:24 noc3 kernel: [36240.776083] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.776091] [] ? wait_on_page_read+0x60/0x60 Nov 18 20:11:24 noc3 kernel: [36240.776095] [] io_schedule+0x9d/0x130 Nov 18 20:11:24 noc3 kernel: [36240.776097] [] sleep_on_page+0xe/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776101] [] wait_on_bit_lock+0x5b/0xc0 Nov 18 20:11:24 noc3 kernel: [36240.776103] [] lock_page+0x6a/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776107] [] ? wake_atomic_t_function+0x40/0x40 Nov 18 20:11:24 noc3 kernel: [36240.776109] [] ? find_get_page+0x1e/0xa0 Nov 18 20:11:24 noc3 kernel: [36240.776112] [] lock_page_or_retry+0x5d/0xc0 Nov 18 20:11:24 noc3 kernel: [36240.776116] [] handle_pte_fault+0xa75/0xab0 Nov 18 20:11:24 noc3 kernel: [36240.776120] [] ? find_busiest_group+0x36/0x4a0 Nov 18 20:11:24 noc3 kernel: [36240.776122] [] handle_mm_fault+0x299/0x670 Nov 18 20:11:24 noc3 kernel: [36240.776125] [] ? _raw_spin_lock+0xe/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776128] [] __get_user_pages+0x166/0x5d0 Nov 18 20:11:24 noc3 kernel: [36240.776131] [] get_user_pages+0x52/0x60 Nov 18 20:11:24 noc3 kernel: [36240.776141] [] async_pf_execute+0x82/0x180 [kvm] Nov 18 20:11:24 noc3 kernel: [36240.776145] [] process_one_work+0x17c/0x430 Nov 18 20:11:24 noc3 kernel: [36240.776147] [] worker_thread+0x11c/0x3c0 Nov 18 20:11:24 noc3 kernel: [36240.776150] [] ? manage_workers.isra.24+0x2a0/0x2a0 Nov 18 20:11:24 noc3 kernel: [36240.776153] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.776156] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776159] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.776162] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776168] INFO: task kworker/0:1:74 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.776169] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.776171] kworker/0:1 D ffff88041fc14580 0 74 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.776182] Workqueue: events async_pf_execute [kvm] Nov 18 20:11:24 noc3 kernel: [36240.776184] ffff88040927db28 0000000000000046 ffff88040927dfd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.776187] ffff88040927dfd8 0000000000014580 ffff880409281770 ffff88041fc14e28 Nov 18 20:11:24 noc3 kernel: [36240.776190] ffff88040927dbb0 0000000000000002 ffffffff8113ead0 ffff88040927dba0 Nov 18 20:11:24 noc3 kernel: [36240.776192] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.776195] [] ? wait_on_page_read+0x60/0x60 Nov 18 20:11:24 noc3 kernel: [36240.776198] [] io_schedule+0x9d/0x130 Nov 18 20:11:24 noc3 kernel: [36240.776200] [] sleep_on_page+0xe/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776203] [] wait_on_bit_lock+0x5b/0xc0 Nov 18 20:11:24 noc3 kernel: [36240.776205] [] lock_page+0x6a/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776207] [] ? wake_atomic_t_function+0x40/0x40 Nov 18 20:11:24 noc3 kernel: [36240.776210] [] ? find_get_page+0x1e/0xa0 Nov 18 20:11:24 noc3 kernel: [36240.776212] [] lock_page_or_retry+0x5d/0xc0 Nov 18 20:11:24 noc3 kernel: [36240.776215] [] handle_pte_fault+0xa75/0xab0 Nov 18 20:11:24 noc3 kernel: [36240.776217] [] ? find_busiest_group+0x36/0x4a0 Nov 18 20:11:24 noc3 kernel: [36240.776220] [] handle_mm_fault+0x299/0x670 Nov 18 20:11:24 noc3 kernel: [36240.776223] [] ? _raw_spin_lock+0xe/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776225] [] get_user_pages+0x166/0x5d0 Nov 18 20:11:24 noc3 kernel: [36240.776228] [] ? dequeue_entity+0x143/0x5d0 Nov 18 20:11:24 noc3 kernel: [36240.776230] [] get_user_pages+0x52/0x60 Nov 18 20:11:24 noc3 kernel: [36240.776240] [] async_pf_execute+0x82/0x180 [kvm] Nov 18 20:11:24 noc3 kernel: [36240.776243] [] process_one_work+0x17c/0x430 Nov 18 20:11:24 noc3 kernel: [36240.776246] [] worker_thread+0x11c/0x3c0 Nov 18 20:11:24 noc3 kernel: [36240.776248] [] ? manage_workers.isra.24+0x2a0/0x2a0 Nov 18 20:11:24 noc3 kernel: [36240.776251] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.776254] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776256] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.776259] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776261] INFO: task kworker/1:1:75 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.776262] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.776264] kworker/1:1 D ffff88041fc54580 0 75 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.776275] Workqueue: events async_pf_execute [kvm] Nov 18 20:11:24 noc3 kernel: [36240.776276] ffff8804092afb28 0000000000000046 ffff8804092affd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.776279] ffff8804092affd8 0000000000014580 ffff880409282ee0 ffff88041fc54e28 Nov 18 20:11:24 noc3 kernel: [36240.776282] ffff8804092afbb0 0000000000000002 ffffffff8113ead0 ffff8804092afba0 Nov 18 20:11:24 noc3 kernel: [36240.776285] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.776287] [] ? wait_on_page_read+0x60/0x60 Nov 18 20:11:24 noc3 kernel: [36240.776290] [] io_schedule+0x9d/0x130 Nov 18 20:11:24 noc3 kernel: [36240.776292] [] sleep_on_page+0xe/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776295] [] wait_on_bit_lock+0x5b/0xc0 Nov 18 20:11:24 noc3 kernel: [36240.776297] [] lock_page+0x6a/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776300] [] ? wake_atomic_t_function+0x40/0x40 Nov 18 20:11:24 noc3 kernel: [36240.776302] [] ? find_get_page+0x1e/0xa0 Nov 18 20:11:24 noc3 kernel: [36240.776304] [] lock_page_or_retry+0x5d/0xc0 Nov 18 20:11:24 noc3 kernel: [36240.776307] [] handle_pte_fault+0xa75/0xab0 Nov 18 20:11:24 noc3 kernel: [36240.776310] [] ? find_busiest_group+0x36/0x4a0 Nov 18 20:11:24 noc3 kernel: [36240.776312] [] handle_mm_fault+0x299/0x670 Nov 18 20:11:24 noc3 kernel: [36240.776315] [] ? _raw_spin_lock+0xe/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776318] [] get_user_pages+0x166/0x5d0 Nov 18 20:11:24 noc3 kernel: [36240.776320] [] get_user_pages+0x52/0x60 Nov 18 20:11:24 noc3 kernel: [36240.776330] [] async_pf_execute+0x82/0x180 [kvm] Nov 18 20:11:24 noc3 kernel: [36240.776333] [] process_one_work+0x17c/0x430 Nov 18 20:11:24 noc3 kernel: [36240.776335] [] worker_thread+0x11c/0x3c0 Nov 18 20:11:24 noc3 kernel: [36240.776338] [] ? manage_workers.isra.24+0x2a0/0x2a0 Nov 18 20:11:24 noc3 kernel: [36240.776341] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.776344] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776346] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.776349] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776352] INFO: task kswapd0:83 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.776353] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.776354] kswapd0 D ffff88041fc14580 0 83 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.776357] ffff880408bc77c0 0000000000000046 ffff880408bc7fd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.776360] ffff880408bc7fd8 0000000000014580 ffff8804050d9770 ffff88041fc14e28 Nov 18 20:11:24 noc3 kernel: [36240.776363] ffff88021a456c00 ffff8803ee2e0000 ffff8803ee2e0030 ffff8803ee2e0030 Nov 18 20:11:24 noc3 kernel: [36240.776365] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.776368] [] io_schedule+0x9d/0x130 Nov 18 20:11:24 noc3 kernel: [36240.776372] [] get_request+0x199/0x6f0 Nov 18 20:11:24 noc3 kernel: [36240.776375] [] ? wake_up_atomic_t+0x30/0x30 Nov 18 20:11:24 noc3 kernel: [36240.776378] [] blk_queue_bio+0x89/0x3c0 Nov 18 20:11:24 noc3 kernel: [36240.776381] [] generic_make_request+0xc2/0x110 Nov 18 20:11:24 noc3 kernel: [36240.776383] [] submit_bio+0x71/0x150 Nov 18 20:11:24 noc3 kernel: [36240.776387] [] ? test_set_page_writeback+0xb2/0x150 Nov 18 20:11:24 noc3 kernel: [36240.776390] [] swap_writepage+0x194/0x240 Nov 18 20:11:24 noc3 kernel: [36240.776394] [] ? mmu_notifier_invalidate_page+0x67/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776397] [] ? try_to_unmap_one+0x3c8/0x440 Nov 18 20:11:24 noc3 kernel: [36240.776399] [] ? _raw_spin_lock+0xe/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776402] [] ? swap_info_get+0x5d/0xe0 Nov 18 20:11:24 noc3 kernel: [36240.776405] [] swap_writepage+0x39/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776408] [] shrink_page_list+0x7b8/0xa40 Nov 18 20:11:24 noc3 kernel: [36240.776411] [] shrink_inactive_list+0x19b/0x500 Nov 18 20:11:24 noc3 kernel: [36240.776414] [] shrink_lruvec+0x345/0x680 Nov 18 20:11:24 noc3 kernel: [36240.776417] [] shrink_zone+0x66/0x1a0 Nov 18 20:11:24 noc3 kernel: [36240.776419] [] balance_pgdat+0x46c/0x5c0 Nov 18 20:11:24 noc3 kernel: [36240.776422] [] kswapd+0x15b/0x410 Nov 18 20:11:24 noc3 kernel: [36240.776424] [] ? wake_up_atomic_t+0x30/0x30 Nov 18 20:11:24 noc3 kernel: [36240.776427] [] ? balance_pgdat+0x5c0/0x5c0 Nov 18 20:11:24 noc3 kernel: [36240.776429] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.776432] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776434] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.776437] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776443] INFO: task spl_kmem_cache/:300 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.776444] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.776445] spl_kmem_cache/ D ffff88041fc14580 0 300 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.776448] ffff8804084653f8 0000000000000046 ffff880408465fd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.776451] ffff880408465fd8 0000000000014580 ffff8804048aaee0 ffff88041fc14e28 Nov 18 20:11:24 noc3 kernel: [36240.776454] ffff8802fe4456c0 ffff8803ee2e0000 ffff8803ee2e0030 ffff8803ee2e0030 Nov 18 20:11:24 noc3 kernel: [36240.776456] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.776459] [] io_schedule+0x9d/0x130 Nov 18 20:11:24 noc3 kernel: [36240.776462] [] get_request+0x199/0x6f0 Nov 18 20:11:24 noc3 kernel: [36240.776464] [] ? wake_up_atomic_t+0x30/0x30 Nov 18 20:11:24 noc3 kernel: [36240.776467] [] blk_queue_bio+0x89/0x3c0 Nov 18 20:11:24 noc3 kernel: [36240.776470] [] generic_make_request+0xc2/0x110 Nov 18 20:11:24 noc3 kernel: [36240.776472] [] submit_bio+0x71/0x150 Nov 18 20:11:24 noc3 kernel: [36240.776475] [] ? test_set_page_writeback+0xb2/0x150 Nov 18 20:11:24 noc3 kernel: [36240.776477] [] __swap_writepage+0x194/0x240 Nov 18 20:11:24 noc3 kernel: [36240.776480] [] ? mmu_notifier_invalidate_page+0x67/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776483] [] ? try_to_unmap_one+0x3c8/0x440 Nov 18 20:11:24 noc3 kernel: [36240.776485] [] ? _raw_spin_lock+0xe/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776488] [] ? swap_info_get+0x5d/0xe0 Nov 18 20:11:24 noc3 kernel: [36240.776490] [] swap_writepage+0x39/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776493] [] shrink_page_list+0x7b8/0xa40 Nov 18 20:11:24 noc3 kernel: [36240.776496] [] shrink_inactive_list+0x19b/0x500 Nov 18 20:11:24 noc3 kernel: [36240.776498] [] shrink_lruvec+0x345/0x680 Nov 18 20:11:24 noc3 kernel: [36240.776501] [] shrink_zone+0x66/0x1a0 Nov 18 20:11:24 noc3 kernel: [36240.776504] [] do_try_to_free_pages+0xe0/0x4c0 Nov 18 20:11:24 noc3 kernel: [36240.776506] [] ? get_page_from_freelist+0x55c/0x870 Nov 18 20:11:24 noc3 kernel: [36240.776509] [] try_to_free_pages+0xd1/0x160 Nov 18 20:11:24 noc3 kernel: [36240.776512] [] alloc_pages_nodemask+0x5a9/0x920 Nov 18 20:11:24 noc3 kernel: [36240.776515] [] alloc_pages_current+0xa9/0x160 Nov 18 20:11:24 noc3 kernel: [36240.776517] [] get_free_pages+0xe/0x50 Nov 18 20:11:24 noc3 kernel: [36240.776521] [] pte_alloc_one_kernel+0x15/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776524] [] pte_alloc_kernel+0x1d/0xf0 Nov 18 20:11:24 noc3 kernel: [36240.776526] [] vmap_page_range_noflush+0x307/0x370 Nov 18 20:11:24 noc3 kernel: [36240.776529] [] map_vm_area+0x2e/0x40 Nov 18 20:11:24 noc3 kernel: [36240.776531] [] __vmalloc_node_range+0x170/0x250 Nov 18 20:11:24 noc3 kernel: [36240.776546] [] ? kv_alloc.isra.9+0x49/0x50 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776549] [] vmalloc+0x41/0x50 Nov 18 20:11:24 noc3 kernel: [36240.776555] [] ? kv_alloc.isra.9+0x49/0x50 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776562] [] kv_alloc.isra.9+0x49/0x50 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776569] [] spl_cache_grow_work+0x40/0x460 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776573] [] ? wake_up+0x44/0x50 Nov 18 20:11:24 noc3 kernel: [36240.776580] [] taskq_thread+0x237/0x4b0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776583] [] ? finish_task_switch+0x50/0xf0 Nov 18 20:11:24 noc3 kernel: [36240.776587] [] ? wake_up_state+0x20/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776594] [] ? taskq_cancel_id+0x1f0/0x1f0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776597] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.776600] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776602] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.776605] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776608] INFO: task arc_adapt:309 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.776609] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.776611] arc_adapt D ffff88041fd54580 0 309 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.776613] ffff8804066a7da0 0000000000000046 ffff8804066a7fd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.776616] ffff8804066a7fd8 0000000000014580 ffff8804048a9770 ffffffffa02fe520 Nov 18 20:11:24 noc3 kernel: [36240.776619] ffffffffa02fe524 ffff8804048a9770 00000000ffffffff ffffffffa02fe528 Nov 18 20:11:24 noc3 kernel: [36240.776622] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.776626] [] schedule_preempt_disabled+0x29/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776629] [] mutex_lock_slowpath+0x13f/0x1c0 Nov 18 20:11:24 noc3 kernel: [36240.776633] [] ? mutex_lock+0x12/0x2f Nov 18 20:11:24 noc3 kernel: [36240.776636] [] mutex_lock+0x1f/0x2f Nov 18 20:11:24 noc3 kernel: [36240.776676] [] dbuf_destroy+0xe9/0x170 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776693] [] dbuf_do_evict+0x4a/0x70 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776709] [] arc_do_user_evicts+0x50/0xd0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776725] [] ? arc_adjust_meta+0x1d0/0x1d0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776741] [] arc_adapt_thread+0x7b/0x170 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776748] [] thread_generic_wrapper+0x7a/0x90 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776755] [] ? thread_exit+0xa0/0xa0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776758] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.776761] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776764] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.776766] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776769] INFO: task zvol/3:315 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.776771] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.776772] zvol/3 D ffff88041fcd4580 0 315 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.776774] ffff8804066dba70 0000000000000046 ffff8804066dbfd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.776777] ffff8804066dbfd8 0000000000014580 ffff8804066cddc0 ffff88025ec48ad8 Nov 18 20:11:24 noc3 kernel: [36240.776780] ffff88025ec48adc ffff8804066cddc0 00000000ffffffff ffff88025ec48ae0 Nov 18 20:11:24 noc3 kernel: [36240.776783] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.776786] [] schedule_preempt_disabled+0x29/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776789] [] __mutex_lock_slowpath+0x13f/0x1c0 Nov 18 20:11:24 noc3 kernel: [36240.776791] [] ? mutex_lock+0x12/0x2f Nov 18 20:11:24 noc3 kernel: [36240.776795] [] mutex_lock+0x1f/0x2f Nov 18 20:11:24 noc3 kernel: [36240.776812] [] dbuf_create+0x1c4/0x390 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776829] [] dbuf_hold_impl+0x1b4/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776846] [] dbuf_hold_impl+0x168/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776863] [] dbuf_hold_impl+0x7b/0xa0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776884] [] dmu_tx_count_write+0x3c7/0x6f0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776905] [] dmu_tx_hold_write+0x36/0x50 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776936] [] zvol_write+0x9a/0x480 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.776943] [] taskq_thread+0x237/0x4b0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776946] [] ? finish_task_switch+0x50/0xf0 Nov 18 20:11:24 noc3 kernel: [36240.776949] [] ? wake_up_state+0x20/0x20 Nov 18 20:11:24 noc3 kernel: [36240.776956] [] ? taskq_cancel_id+0x1f0/0x1f0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.776959] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.776962] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776964] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.776967] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.776970] INFO: task zvol/8:320 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.776971] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.776972] zvol/8 D ffff88041fc14580 0 320 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.776975] ffff8804066f5a20 0000000000000046 ffff8804066f5fd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.776977] ffff8804066f5fd8 0000000000014580 ffff8804066eddc0 ffff8803ec4e1a08 Nov 18 20:11:24 noc3 kernel: [36240.776980] ffff8803ec4e1a0c ffff8804066eddc0 00000000ffffffff ffff8803ec4e1a10 Nov 18 20:11:24 noc3 kernel: [36240.776983] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.776986] [] schedule_preempt_disabled+0x29/0x70 Nov 18 20:11:24 noc3 kernel: [36240.776989] [] mutex_lock_slowpath+0x13f/0x1c0 Nov 18 20:11:24 noc3 kernel: [36240.776992] [] mutex_lock+0x1f/0x2f Nov 18 20:11:24 noc3 kernel: [36240.777008] [] dbuf_create+0xf0/0x390 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777025] [] ? dbuf_hold_impl+0x1fb/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777043] [] dbuf_hold_impl+0x1b4/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777060] [] dbuf_hold_impl+0x168/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777076] [] __dbuf_hold_impl+0x168/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777094] [] dbuf_hold_impl+0x7b/0xa0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777114] [] dmu_tx_count_write+0x3c7/0x6f0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777134] [] dmu_tx_hold_write+0x36/0x50 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777159] [] zvol_write+0x9a/0x480 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777166] [] taskq_thread+0x237/0x4b0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.777169] [] ? finish_task_switch+0x50/0xf0 Nov 18 20:11:24 noc3 kernel: [36240.777172] [] ? wake_up_state+0x20/0x20 Nov 18 20:11:24 noc3 kernel: [36240.777179] [] ? taskq_cancel_id+0x1f0/0x1f0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.777182] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.777185] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.777187] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.777190] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.777192] INFO: task zvol/10:322 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.777193] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.777194] zvol/10 D ffff88041fc94580 0 322 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.777197] ffff880406701a70 0000000000000046 ffff880406701fd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.777200] ffff880406701fd8 0000000000014580 ffff8804066f9770 ffff8803ec4e1a08 Nov 18 20:11:24 noc3 kernel: [36240.777202] ffff8803ec4e1a0c ffff8804066f9770 00000000ffffffff ffff8803ec4e1a10 Nov 18 20:11:24 noc3 kernel: [36240.777205] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.777208] [] schedule_preempt_disabled+0x29/0x70 Nov 18 20:11:24 noc3 kernel: [36240.777211] [] mutex_lock_slowpath+0x13f/0x1c0 Nov 18 20:11:24 noc3 kernel: [36240.777214] [] ? mutex_lock+0x12/0x2f Nov 18 20:11:24 noc3 kernel: [36240.777216] [] mutex_lock+0x1f/0x2f Nov 18 20:11:24 noc3 kernel: [36240.777233] [] dbuf_create+0xf0/0x390 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777250] [] ? dbuf_hold_impl+0x1fb/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777267] [] dbuf_hold_impl+0x1b4/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777284] [] dbuf_hold_impl+0x168/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777302] [] dbuf_hold_impl+0x7b/0xa0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777322] [] dmu_tx_count_write+0x3c7/0x6f0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777342] [] dmu_tx_hold_write+0x36/0x50 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777366] [] zvol_write+0x9a/0x480 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777374] [] taskq_thread+0x237/0x4b0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.777377] [] ? finish_task_switch+0x50/0xf0 Nov 18 20:11:24 noc3 kernel: [36240.777379] [] ? wake_up_state+0x20/0x20 Nov 18 20:11:24 noc3 kernel: [36240.777386] [] ? taskq_cancel_id+0x1f0/0x1f0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.777389] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.777392] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.777395] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.777397] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.777400] INFO: task zvol/17:329 blocked for more than 120 seconds. Nov 18 20:11:24 noc3 kernel: [36240.777402] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Nov 18 20:11:24 noc3 kernel: [36240.777403] zvol/17 D ffff88041fc54580 0 329 2 0x00000000 Nov 18 20:11:24 noc3 kernel: [36240.777405] ffff880406717a70 0000000000000046 ffff880406717fd8 0000000000014580 Nov 18 20:11:24 noc3 kernel: [36240.777408] ffff880406717fd8 0000000000014580 ffff88040670c650 ffff8803ec4e1a08 Nov 18 20:11:24 noc3 kernel: [36240.777411] ffff8803ec4e1a0c ffff88040670c650 00000000ffffffff ffff8803ec4e1a10 Nov 18 20:11:24 noc3 kernel: [36240.777414] Call Trace: Nov 18 20:11:24 noc3 kernel: [36240.777417] [] schedule_preempt_disabled+0x29/0x70 Nov 18 20:11:24 noc3 kernel: [36240.777420] [] __mutex_lock_slowpath+0x13f/0x1c0 Nov 18 20:11:24 noc3 kernel: [36240.777422] [] ? mutex_lock+0x1f/0x2f Nov 18 20:11:24 noc3 kernel: [36240.777425] [] mutex_lock+0x1f/0x2f Nov 18 20:11:24 noc3 kernel: [36240.777442] [] dbuf_create+0xf0/0x390 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777459] [] ? dbuf_hold_impl+0x1fb/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777476] [] dbuf_hold_impl+0x1b4/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777493] [] dbuf_hold_impl+0x168/0x4a0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777510] [] dbuf_hold_impl+0x7b/0xa0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777530] [] dmu_tx_count_write+0x3c7/0x6f0 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777551] [] dmu_tx_hold_write+0x36/0x50 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777575] [] zvol_write+0x9a/0x480 [zfs] Nov 18 20:11:24 noc3 kernel: [36240.777582] [] taskq_thread+0x237/0x4b0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.777585] [] ? finish_task_switch+0x50/0xf0 Nov 18 20:11:24 noc3 kernel: [36240.777588] [] ? wake_up_state+0x20/0x20 Nov 18 20:11:24 noc3 kernel: [36240.777595] [] ? taskq_cancel_id+0x1f0/0x1f0 [spl] Nov 18 20:11:24 noc3 kernel: [36240.777598] [] kthread+0xc0/0xd0 Nov 18 20:11:24 noc3 kernel: [36240.777601] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:11:24 noc3 kernel: [36240.777603] [] ret_from_fork+0x7c/0xb0 Nov 18 20:11:24 noc3 kernel: [36240.777606] [] ? kthread_create_on_node+0x120/0x120 Nov 18 20:15:01 noc3 CRON[18562]: (root) CMD (zfs-auto-snapshot -q -g --label=frequent --keep=4 //) ...

ryao commented 10 years ago

@maci0 @hcoin #2484 might help here.

behlendorf commented 8 years ago

Closing as stale, refer to the swap section of the wiki for additional configuration information.