koptein
commented
8 years ago I am experiencing a similar behavior of zfs on my Ubuntu 14.04.3 File server: I see the txg_sync and several nfsd processes fighting for 100% CPU time combined with unresponsive userland processes. This all happens since the last zfs Update from http://ppa.launchpad.net/zfs-native/stable/ubuntu Here are the details of the machine running 0.6.5.3-1~trusty:
# cat /proc/spl/kstat/zfs/arcstats
6 1 0x01 91 4368 5625952810 159446143340616
name type data
hits 4 1556458151
misses 4 18253654
demand_data_hits 4 751852
demand_data_misses 4 761810
demand_metadata_hits 4 839960418
demand_metadata_misses 4 6659089
prefetch_data_hits 4 72518
prefetch_data_misses 4 213736
prefetch_metadata_hits 4 715673363
prefetch_metadata_misses 4 10619019
mru_hits 4 57211460
mru_ghost_hits 4 1513179
mfu_hits 4 783574541
mfu_ghost_hits 4 218089
deleted 4 14878869
mutex_miss 4 44460
evict_skip 4 5551288181
evict_not_enough 4 6771832
evict_l2_cached 4 0
evict_l2_eligible 4 145399854592
evict_l2_ineligible 4 112644096512
evict_l2_skip 4 0
hash_elements 4 772782
hash_elements_max 4 1242299
hash_collisions 4 1772716
hash_chains 4 33599
hash_chain_max 4 4
p 4 31652634240
c 4 33762809856
c_min 4 33554432
c_max 4 33762809856
size 4 25336923096
hdr_size 4 327689040
data_size 4 151552
metadata_size 4 12499697152
other_size 4 12509385352
anon_size 4 8509440
anon_evictable_data 4 0
anon_evictable_metadata 4 0
mru_size 4 146642944
mru_evictable_data 4 0
mru_evictable_metadata 4 21987328
mru_ghost_size 4 0
mru_ghost_evictable_data 4 0
mru_ghost_evictable_metadata 4 0
mfu_size 4 12344696320
mfu_evictable_data 4 0
mfu_evictable_metadata 4 12027229696
mfu_ghost_size 4 0
mfu_ghost_evictable_data 4 0
mfu_ghost_evictable_metadata 4 0
l2_hits 4 0
l2_misses 4 0
l2_feeds 4 0
l2_rw_clash 4 0
l2_read_bytes 4 0
l2_write_bytes 4 0
l2_writes_sent 4 0
l2_writes_done 4 0
l2_writes_error 4 0
l2_writes_lock_retry 4 0
l2_evict_lock_retry 4 0
l2_evict_reading 4 0
l2_evict_l1cached 4 0
l2_free_on_write 4 0
l2_cdata_free_on_write 4 0
l2_abort_lowmem 4 0
l2_cksum_bad 4 0
l2_io_error 4 0
l2_size 4 0
l2_asize 4 0
l2_hdr_size 4 0
l2_compress_successes 4 0
l2_compress_zeros 4 0
l2_compress_failures 4 0
memory_throttle_count 4 0
duplicate_buffers 4 0
duplicate_buffers_size 4 0
duplicate_reads 4 2
memory_direct_count 4 0
memory_indirect_count 4 0
arc_no_grow 4 0
arc_tempreserve 4 0
arc_loaned_bytes 4 0
arc_prune 4 36012386
arc_meta_used 4 25336771544
arc_meta_limit 4 25322107392
arc_meta_max 4 25391141688
arc_meta_min 4 16777216
arc_need_free 4 0
arc_sys_free 4 1055084544# free -m
total used free shared buffers cached
Mem: 64397 34931 29465 0 248 407
-/+ buffers/cache: 34276 30121
Swap: 5719 0 5719# cat /var/log/kern.log
[ 0.591756] pci_bus 0000:00: root bus resource [bus 00-fe]
[ 0.591758] pci_bus 0000:00: root bus resource [io 0x0000-0x03af]
[ 0.591760] pci_bus 0000:00: root bus resource [io 0x03e0-0x0cf7]
[ 0.591763] pci_bus 0000:00: root bus resource [io 0x03b0-0x03df]
[ 0.591765] pci_bus 0000:00: root bus resource [io 0x0d00-0xffff]
[ 0.591767] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff]
[ 0.591769] pci_bus 0000:00: root bus resource [mem 0x000c0000-0x000dffff]
[ 0.591772] pci_bus 0000:00: root bus resource [mem 0xfed08000-0xfed08fff]
[ 0.591774] pci_bus 0000:00: root bus resource [mem 0xfed0e000-0xfed0ffff]
[ 0.591776] pci_bus 0000:00: root bus resource [mem 0x80000000-0xfbffffff]
[ 0.591788] pci 0000:00:00.0: [8086:0e00] type 00 class 0x060000
[ 0.591845] pci 0000:00:00.0: PME# supported from D0 D3hot D3cold
[ 0.591918] pci 0000:00:01.0: [8086:0e02] type 01 class 0x060400
[ 0.591982] pci 0000:00:01.0: PME# supported from D0 D3hot D3cold
[ 0.592030] pci 0000:00:01.0: System wakeup disabled by ACPI
[ 0.592062] pci 0000:00:02.0: [8086:0e04] type 01 class 0x060400
[ 0.592124] pci 0000:00:02.0: PME# supported from D0 D3hot D3cold
[ 0.592170] pci 0000:00:02.0: System wakeup disabled by ACPI
[ 0.592199] pci 0000:00:02.2: [8086:0e06] type 01 class 0x060400
[ 0.592261] pci 0000:00:02.2: PME# supported from D0 D3hot D3cold
[ 0.592307] pci 0000:00:02.2: System wakeup disabled by ACPI
[ 0.592339] pci 0000:00:03.0: [8086:0e08] type 01 class 0x060400
[ 0.592404] pci 0000:00:03.0: PME# supported from D0 D3hot D3cold
[ 0.592450] pci 0000:00:03.0: System wakeup disabled by ACPI
[ 0.592477] pci 0000:00:03.1: [8086:0e09] type 01 class 0x060400
[ 0.592539] pci 0000:00:03.1: PME# supported from D0 D3hot D3cold
[ 0.592588] pci 0000:00:03.1: System wakeup disabled by ACPI
[ 0.592616] pci 0000:00:03.2: [8086:0e0a] type 01 class 0x060400
[ 0.592678] pci 0000:00:03.2: PME# supported from D0 D3hot D3cold
[ 0.592724] pci 0000:00:03.2: System wakeup disabled by ACPI
[ 0.592751] pci 0000:00:04.0: [8086:0e20] type 00 class 0x088000
[ 0.592766] pci 0000:00:04.0: reg 0x10: [mem 0xfbd1c000-0xfbd1ffff 64bit]
[ 0.592883] pci 0000:00:04.1: [8086:0e21] type 00 class 0x088000
[ 0.592896] pci 0000:00:04.1: reg 0x10: [mem 0xfbd18000-0xfbd1bfff 64bit]
[ 0.593013] pci 0000:00:04.2: [8086:0e22] type 00 class 0x088000
[ 0.593026] pci 0000:00:04.2: reg 0x10: [mem 0xfbd14000-0xfbd17fff 64bit]
[ 0.593142] pci 0000:00:04.3: [8086:0e23] type 00 class 0x088000
[ 0.593155] pci 0000:00:04.3: reg 0x10: [mem 0xfbd10000-0xfbd13fff 64bit]
[ 0.593272] pci 0000:00:04.4: [8086:0e24] type 00 class 0x088000
[ 0.593285] pci 0000:00:04.4: reg 0x10: [mem 0xfbd0c000-0xfbd0ffff 64bit]
[ 0.593400] pci 0000:00:04.5: [8086:0e25] type 00 class 0x088000
[ 0.593413] pci 0000:00:04.5: reg 0x10: [mem 0xfbd08000-0xfbd0bfff 64bit]
[ 0.593529] pci 0000:00:04.6: [8086:0e26] type 00 class 0x088000
[ 0.593542] pci 0000:00:04.6: reg 0x10: [mem 0xfbd04000-0xfbd07fff 64bit]
[ 0.593659] pci 0000:00:04.7: [8086:0e27] type 00 class 0x088000
[ 0.593672] pci 0000:00:04.7: reg 0x10: [mem 0xfbd00000-0xfbd03fff 64bit]
[ 0.593787] pci 0000:00:05.0: [8086:0e28] type 00 class 0x088000
[ 0.593889] pci 0000:00:05.2: [8086:0e2a] type 00 class 0x088000
[ 0.593990] pci 0000:00:05.4: [8086:0e2c] type 00 class 0x080020
[ 0.593999] pci 0000:00:05.4: reg 0x10: [mem 0xfbd25000-0xfbd25fff]
[ 0.594124] pci 0000:00:11.0: [8086:1d3e] type 01 class 0x060400
[ 0.594216] pci 0000:00:11.0: PME# supported from D0 D3hot D3cold
[ 0.594301] pci 0000:00:16.0: [8086:1d3a] type 00 class 0x078000
[ 0.594323] pci 0000:00:16.0: reg 0x10: [mem 0xfed0e000-0xfed0e00f 64bit]
[ 0.594393] pci 0000:00:16.0: PME# supported from D0 D3hot D3cold
[ 0.594459] pci 0000:00:16.1: [8086:1d3b] type 00 class 0x078000
[ 0.594480] pci 0000:00:16.1: reg 0x10: [mem 0xfed0f000-0xfed0f00f 64bit]
[ 0.594550] pci 0000:00:16.1: PME# supported from D0 D3hot D3cold
[ 0.594628] pci 0000:00:1a.0: [8086:1d2d] type 00 class 0x0c0320
[ 0.594648] pci 0000:00:1a.0: reg 0x10: [mem 0xfbd23000-0xfbd233ff]
[ 0.594737] pci 0000:00:1a.0: PME# supported from D0 D3hot D3cold
[ 0.594783] pci 0000:00:1a.0: System wakeup disabled by ACPI
[ 0.594817] pci 0000:00:1c.0: [8086:1d10] type 01 class 0x060400
[ 0.594945] pci 0000:00:1c.0: PME# supported from D0 D3hot D3cold
[ 0.594999] pci 0000:00:1c.0: System wakeup disabled by ACPI
[ 0.595032] pci 0000:00:1c.6: [8086:1d1c] type 01 class 0x060400
[ 0.595109] pci 0000:00:1c.6: PME# supported from D0 D3hot D3cold
[ 0.595155] pci 0000:00:1c.6: System wakeup disabled by ACPI
[ 0.595182] pci 0000:00:1c.7: [8086:1d1e] type 01 class 0x060400
[ 0.595259] pci 0000:00:1c.7: PME# supported from D0 D3hot D3cold
[ 0.595305] pci 0000:00:1c.7: System wakeup disabled by ACPI
[ 0.595336] pci 0000:00:1d.0: [8086:1d26] type 00 class 0x0c0320
[ 0.595356] pci 0000:00:1d.0: reg 0x10: [mem 0xfbd22000-0xfbd223ff]
[ 0.595445] pci 0000:00:1d.0: PME# supported from D0 D3hot D3cold
[ 0.595489] pci 0000:00:1d.0: System wakeup disabled by ACPI
[ 0.595514] pci 0000:00:1e.0: [8086:244e] type 01 class 0x060401
[ 0.595598] pci 0000:00:1e.0: System wakeup disabled by ACPI
[ 0.595625] pci 0000:00:1f.0: [8086:1d41] type 00 class 0x060100
[ 0.595792] pci 0000:00:1f.2: [8086:1d02] type 00 class 0x010601
[ 0.595810] pci 0000:00:1f.2: reg 0x10: [io 0xf050-0xf057]
[ 0.595817] pci 0000:00:1f.2: reg 0x14: [io 0xf040-0xf043]
[ 0.595824] pci 0000:00:1f.2: reg 0x18: [io 0xf030-0xf037]
[ 0.595832] pci 0000:00:1f.2: reg 0x1c: [io 0xf020-0xf023]
[ 0.595839] pci 0000:00:1f.2: reg 0x20: [io 0xf000-0xf01f]
[ 0.595846] pci 0000:00:1f.2: reg 0x24: [mem 0xfbd21000-0xfbd217ff]
[ 0.595891] pci 0000:00:1f.2: PME# supported from D3hot
[ 0.595951] pci 0000:00:1f.3: [8086:1d22] type 00 class 0x0c0500
[ 0.595966] pci 0000:00:1f.3: reg 0x10: [mem 0xfbd20000-0xfbd200ff 64bit]
[ 0.595986] pci 0000:00:1f.3: reg 0x20: [io 0x1180-0x119f]
[ 0.596061] pci 0000:00:1f.6: [8086:1d24] type 00 class 0x118000
[ 0.596082] pci 0000:00:1f.6: reg 0x10: [mem 0xfed08000-0xfed08fff 64bit]
[ 0.596234] pci 0000:00:01.0: PCI bridge to [bus 01]
[ 0.596288] pci 0000:02:00.0: [1000:0087] type 00 class 0x010700
[ 0.596295] pci 0000:02:00.0: reg 0x10: [io 0xe000-0xe0ff]
[ 0.596303] pci 0000:02:00.0: reg 0x14: [mem 0xfba40000-0xfba4ffff 64bit]
[ 0.596310] pci 0000:02:00.0: reg 0x1c: [mem 0xfba00000-0xfba3ffff 64bit]
[ 0.596319] pci 0000:02:00.0: reg 0x30: [mem 0xfb900000-0xfb9fffff pref]
[ 0.596347] pci 0000:02:00.0: supports D1 D2
[ 0.602323] pci 0000:00:02.0: PCI bridge to [bus 02]
[ 0.602327] pci 0000:00:02.0: bridge window [io 0xe000-0xefff]
[ 0.602330] pci 0000:00:02.0: bridge window [mem 0xfb900000-0xfbafffff]
[ 0.602379] pci 0000:00:02.2: PCI bridge to [bus 03]
[ 0.602426] pci 0000:00:03.0: PCI bridge to [bus 04]
[ 0.602474] pci 0000:00:03.1: PCI bridge to [bus 05]
[ 0.602521] pci 0000:00:03.2: PCI bridge to [bus 06]
[ 0.602601] pci 0000:07:00.0: [8086:1d6b] type 00 class 0x010700
[ 0.602619] pci 0000:07:00.0: reg 0x10: [mem 0xfa47c000-0xfa47ffff 64bit pref]
[ 0.602634] pci 0000:07:00.0: reg 0x18: [mem 0xfa000000-0xfa3fffff 64bit pref]
[ 0.602644] pci 0000:07:00.0: reg 0x20: [io 0xd000-0xd0ff]
[ 0.602745] pci 0000:07:00.0: reg 0x164: [mem 0xfa400000-0xfa403fff 64bit pref]
[ 0.602815] pci 0000:00:11.0: PCI bridge to [bus 07]
[ 0.602819] pci 0000:00:11.0: bridge window [io 0xd000-0xdfff]
[ 0.602828] pci 0000:00:11.0: bridge window [mem 0xfa000000-0xfa4fffff 64bit pref]
[ 0.602904] pci 0000:00:1c.0: PCI bridge to [bus 08]
[ 0.603009] pci 0000:09:00.0: [8086:10d3] type 00 class 0x020000
[ 0.603036] pci 0000:09:00.0: reg 0x10: [mem 0xfbc00000-0xfbc1ffff]
[ 0.603073] pci 0000:09:00.0: reg 0x18: [io 0xc000-0xc01f]
[ 0.603092] pci 0000:09:00.0: reg 0x1c: [mem 0xfbc20000-0xfbc23fff]
[ 0.603252] pci 0000:09:00.0: PME# supported from D0 D3hot D3cold
[ 0.603295] pci 0000:09:00.0: System wakeup disabled by ACPI
[ 0.610326] pci 0000:00:1c.6: PCI bridge to [bus 09]
[ 0.610347] pci 0000:00:1c.6: bridge window [io 0xc000-0xcfff]
[ 0.610351] pci 0000:00:1c.6: bridge window [mem 0xfbc00000-0xfbcfffff]
[ 0.610435] pci 0000:0a:00.0: [8086:10d3] type 00 class 0x020000
[ 0.610462] pci 0000:0a:00.0: reg 0x10: [mem 0xfbb00000-0xfbb1ffff]
[ 0.610500] pci 0000:0a:00.0: reg 0x18: [io 0xb000-0xb01f]
[ 0.610520] pci 0000:0a:00.0: reg 0x1c: [mem 0xfbb20000-0xfbb23fff]
[ 0.610679] pci 0000:0a:00.0: PME# supported from D0 D3hot D3cold
[ 0.618348] pci 0000:00:1c.7: PCI bridge to [bus 0a]
[ 0.618352] pci 0000:00:1c.7: bridge window [io 0xb000-0xbfff]
[ 0.618356] pci 0000:00:1c.7: bridge window [mem 0xfbb00000-0xfbbfffff]
[ 0.618400] pci 0000:0b:04.0: [102b:0532] type 00 class 0x030000
[ 0.618416] pci 0000:0b:04.0: reg 0x10: [mem 0xf9000000-0xf9ffffff pref]
[ 0.618425] pci 0000:0b:04.0: reg 0x14: [mem 0xfb800000-0xfb803fff]
[ 0.618434] pci 0000:0b:04.0: reg 0x18: [mem 0xfb000000-0xfb7fffff]
[ 0.618543] pci 0000:00:1e.0: PCI bridge to [bus 0b] (subtractive decode)
[ 0.618549] pci 0000:00:1e.0: bridge window [mem 0xfb000000-0xfb8fffff]
[ 0.618554] pci 0000:00:1e.0: bridge window [mem 0xf9000000-0xf9ffffff 64bit pref]
[ 0.618556] pci 0000:00:1e.0: bridge window [io 0x0000-0x03af] (subtractive decode)
[ 0.618558] pci 0000:00:1e.0: bridge window [io 0x03e0-0x0cf7] (subtractive decode)
[ 0.618559] pci 0000:00:1e.0: bridge window [io 0x03b0-0x03df] (subtractive decode)
[ 0.618561] pci 0000:00:1e.0: bridge window [io 0x0d00-0xffff] (subtractive decode)
[ 0.618562] pci 0000:00:1e.0: bridge window [mem 0x000a0000-0x000bffff] (subtractive decode)
[ 0.618564] pci 0000:00:1e.0: bridge window [mem 0x000c0000-0x000dffff] (subtractive decode)
[ 0.618565] pci 0000:00:1e.0: bridge window [mem 0xfed08000-0xfed08fff] (subtractive decode)
[ 0.618567] pci 0000:00:1e.0: bridge window [mem 0xfed0e000-0xfed0ffff] (subtractive decode)
[ 0.618568] pci 0000:00:1e.0: bridge window [mem 0x80000000-0xfbffffff] (subtractive decode)
[ 0.618627] pci_bus 0000:00: on NUMA node 0 (pxm 0)
[ 0.619147] ACPI: PCI Root Bridge [UNC0] (domain 0000 [bus ff])
[ 0.619151] acpi PNP0A03:00: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI]
[ 0.619172] acpi PNP0A03:00: _OSC: OS now controls [PCIeHotplug PME AER PCIeCapability]
[ 0.619175] acpi PNP0A03:00: FADT indicates ASPM is unsupported, using BIOS configuration
[ 0.619217] PCI host bridge to bus 0000:ff
[ 0.619220] pci_bus 0000:ff: root bus resource [bus ff]
[ 0.619229] pci 0000:ff:08.0: [8086:0e80] type 00 class 0x088000
[ 0.619273] pci 0000:ff:09.0: [8086:0e90] type 00 class 0x088000
[ 0.619316] pci 0000:ff:0a.0: [8086:0ec0] type 00 class 0x088000
[ 0.619351] pci 0000:ff:0a.1: [8086:0ec1] type 00 class 0x088000
[ 0.619386] pci 0000:ff:0a.2: [8086:0ec2] type 00 class 0x088000
[ 0.619422] pci 0000:ff:0a.3: [8086:0ec3] type 00 class 0x088000
[ 0.619457] pci 0000:ff:0b.0: [8086:0e1e] type 00 class 0x088000
[ 0.619490] pci 0000:ff:0b.3: [8086:0e1f] type 00 class 0x088000
[ 0.619524] pci 0000:ff:0c.0: [8086:0ee0] type 00 class 0x088000
[ 0.619558] pci 0000:ff:0c.1: [8086:0ee2] type 00 class 0x088000
[ 0.619592] pci 0000:ff:0c.2: [8086:0ee4] type 00 class 0x088000
[ 0.619627] pci 0000:ff:0d.0: [8086:0ee1] type 00 class 0x088000
[ 0.619661] pci 0000:ff:0d.1: [8086:0ee3] type 00 class 0x088000
[ 0.619697] pci 0000:ff:0d.2: [8086:0ee5] type 00 class 0x088000
[ 0.619732] pci 0000:ff:0e.0: [8086:0ea0] type 00 class 0x088000
[ 0.619769] pci 0000:ff:0e.1: [8086:0e30] type 00 class 0x110100
[ 0.619811] pci 0000:ff:0f.0: [8086:0ea8] type 00 class 0x088000
[ 0.619858] pci 0000:ff:0f.1: [8086:0e71] type 00 class 0x088000
[ 0.619904] pci 0000:ff:0f.2: [8086:0eaa] type 00 class 0x088000
[ 0.619951] pci 0000:ff:0f.3: [8086:0eab] type 00 class 0x088000
[ 0.619997] pci 0000:ff:0f.4: [8086:0eac] type 00 class 0x088000
[ 0.620043] pci 0000:ff:0f.5: [8086:0ead] type 00 class 0x088000
[ 0.620091] pci 0000:ff:10.0: [8086:0eb0] type 00 class 0x088000
[ 0.620138] pci 0000:ff:10.1: [8086:0eb1] type 00 class 0x088000
[ 0.620185] pci 0000:ff:10.2: [8086:0eb2] type 00 class 0x088000
[ 0.620232] pci 0000:ff:10.3: [8086:0eb3] type 00 class 0x088000
[ 0.620280] pci 0000:ff:10.4: [8086:0eb4] type 00 class 0x088000
[ 0.620327] pci 0000:ff:10.5: [8086:0eb5] type 00 class 0x088000
[ 0.620374] pci 0000:ff:10.6: [8086:0eb6] type 00 class 0x088000
[ 0.620423] pci 0000:ff:10.7: [8086:0eb7] type 00 class 0x088000
[ 0.620468] pci 0000:ff:13.0: [8086:0e1d] type 00 class 0x088000
[ 0.620501] pci 0000:ff:13.1: [8086:0e34] type 00 class 0x110100
[ 0.620538] pci 0000:ff:13.4: [8086:0e81] type 00 class 0x088000
[ 0.620572] pci 0000:ff:13.5: [8086:0e36] type 00 class 0x110100
[ 0.620610] pci 0000:ff:16.0: [8086:0ec8] type 00 class 0x088000
[ 0.620643] pci 0000:ff:16.1: [8086:0ec9] type 00 class 0x088000
[ 0.620677] pci 0000:ff:16.2: [8086:0eca] type 00 class 0x088000
[ 0.620719] pci_bus 0000:ff: on NUMA node 0 (pxm 0)
[ 0.620777] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 6 7 10 *11 12 14 15)
[ 0.620822] ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 *5 6 7 10 11 12 14 15)
[ 0.620866] ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 5 6 10 *11 12 14 15)
[ 0.620908] ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 *5 6 10 11 12 14 15)
[ 0.620950] ACPI: PCI Interrupt Link [LNKE] (IRQs 3 4 5 6 7 10 11 12 14 15) *0
[ 0.620993] ACPI: PCI Interrupt Link [LNKF] (IRQs 3 4 5 6 7 10 11 12 14 15) *0
[ 0.621036] ACPI: PCI Interrupt Link [LNKG] (IRQs 3 4 5 6 7 10 11 12 14 15) *0
[ 0.621078] ACPI: PCI Interrupt Link [LNKH] (IRQs 3 4 5 6 *7 10 11 12 14 15)
[ 0.622301] ACPI: Enabled 1 GPEs in block 00 to 3F
[ 0.622332] ACPI: \_SB_.PCI0: notify handler is installed
[ 0.622466] ACPI Error: [\_SB_.PRAD] Namespace lookup failure, AE_NOT_FOUND (20131115/psargs-359)
[ 0.622472] ACPI Error: Method parse/execution failed [\_GPE._L24] (Node ffff881028ef7ac8), AE_NOT_FOUND (20131115/psparse-536)
[ 0.622480] ACPI Exception: AE_NOT_FOUND, while evaluating GPE method [_L24] (20131115/evgpe-580)
[ 0.622489] ACPI: \_SB_.UNC0: notify handler is installed
[ 0.622585] Found 2 acpi root devices
[ 0.622681] vgaarb: setting as boot device: PCI:0000:0b:04.0
[ 0.622684] vgaarb: device added: PCI:0000:0b:04.0,decodes=io+mem,owns=io+mem,locks=none
[ 0.622690] vgaarb: loaded
[ 0.622692] vgaarb: bridge control possible 0000:0b:04.0
[ 0.622828] SCSI subsystem initialized
[ 0.622873] libata version 3.00 loaded.
[ 0.622890] ACPI: bus type USB registered
[ 0.622906] usbcore: registered new interface driver usbfs
[ 0.622914] usbcore: registered new interface driver hub
[ 0.622931] usbcore: registered new device driver usb
[ 0.623029] PCI: Using ACPI for IRQ routing
[ 0.629225] PCI: pci_cache_line_size set to 64 bytes
[ 0.629346] e820: reserve RAM buffer [mem 0x0009ac00-0x0009ffff]
[ 0.629348] e820: reserve RAM buffer [mem 0x7dd27000-0x7fffffff]
[ 0.629422] NetLabel: Initializing
[ 0.629424] NetLabel: domain hash size = 128
[ 0.629425] NetLabel: protocols = UNLABELED CIPSOv4
[ 0.629438] NetLabel: unlabeled traffic allowed by default
[ 0.629487] hpet0: at MMIO 0xfed00000, IRQs 2, 8, 0, 0, 0, 0, 0, 0
[ 0.629493] hpet0: 8 comparators, 64-bit 14.318180 MHz counter
[ 0.631544] Switched to clocksource hpet
[ 0.635650] AppArmor: AppArmor Filesystem Enabled
[ 0.635666] pnp: PnP ACPI init
[ 0.635674] ACPI: bus type PNP registered
[ 0.635736] system 00:00: [mem 0xfc000000-0xfcffffff] has been reserved
[ 0.635739] system 00:00: [mem 0xfd000000-0xfdffffff] has been reserved
[ 0.635742] system 00:00: [mem 0xfe000000-0xfeafffff] has been reserved
[ 0.635744] system 00:00: [mem 0xfeb00000-0xfebfffff] has been reserved
[ 0.635748] system 00:00: Plug and Play ACPI device, IDs PNP0c01 (active)
[ 0.635813] system 00:01: [mem 0xfbffc000-0xfbffdfff] could not be reserved
[ 0.635817] system 00:01: Plug and Play ACPI device, IDs PNP0c02 (active)
[ 0.635827] pnp 00:02: [dma 4]
[ 0.635838] pnp 00:02: Plug and Play ACPI device, IDs PNP0200 (active)
[ 0.635865] pnp 00:03: Plug and Play ACPI device, IDs PNP0b00 (active)
[ 0.635881] pnp 00:04: Plug and Play ACPI device, IDs PNP0800 (active)
[ 0.635934] system 00:05: [io 0x04d0-0x04d1] has been reserved
[ 0.635937] system 00:05: [mem 0x00000400-0x000004ff] could not be reserved
[ 0.635940] system 00:05: Plug and Play ACPI device, IDs PNP0c02 (active)
[ 0.635962] pnp 00:06: Plug and Play ACPI device, IDs PNP0c04 (active)
[ 0.636042] system 00:07: [io 0x0a30-0x0a3f] has been reserved
[ 0.636046] system 00:07: Plug and Play ACPI device, IDs PNP0c02 (active)
[ 0.636102] pnp 00:08: Plug and Play ACPI device, IDs PNP0f03 PNP0f13 (active)
[ 0.636261] pnp 00:09: [dma 0 disabled]
[ 0.636299] pnp 00:09: Plug and Play ACPI device, IDs PNP0501 (active)
[ 0.636449] pnp 00:0a: [dma 0 disabled]
[ 0.636485] pnp 00:0a: Plug and Play ACPI device, IDs PNP0501 (active)
[ 0.636573] system 00:0b: [io 0x0b00-0x0b7f] has been reserved
[ 0.636577] system 00:0b: Plug and Play ACPI device, IDs PNP0c02 (active)
[ 0.636710] pnp 00:0c: [dma 0 disabled]
[ 0.636747] pnp 00:0c: Plug and Play ACPI device, IDs PNP0501 (active)
[ 0.636829] pnp 00:0d: Plug and Play ACPI device, IDs IPI0001 (active)
[ 0.636961] system 00:0e: [io 0x0400-0x0453] could not be reserved
[ 0.636964] system 00:0e: [io 0x0458-0x047f] has been reserved
[ 0.636967] system 00:0e: [io 0x1180-0x119f] has been reserved
[ 0.636969] system 00:0e: [io 0x0500-0x057f] has been reserved
[ 0.636972] system 00:0e: [mem 0xfed1c000-0xfed1ffff] has been reserved
[ 0.636974] system 00:0e: [mem 0xfec00000-0xfecfffff] could not be reserved
[ 0.636977] system 00:0e: [mem 0xff000000-0xffffffff] has been reserved
[ 0.636980] system 00:0e: Plug and Play ACPI device, IDs PNP0c01 (active)
[ 0.637034] system 00:0f: [io 0x0454-0x0457] has been reserved
[ 0.637038] system 00:0f: Plug and Play ACPI device, IDs INT3f0d PNP0c02 (active)
[ 0.637139] pnp 00:10: Plug and Play ACPI device, IDs PNP0103 (active)
[ 0.637271] system 00:11: [mem 0x00000000-0x0009ffff] could not be reserved
[ 0.637275] system 00:11: Plug and Play ACPI device, IDs PNP0c01 (active)
[ 0.637413] pnp: PnP ACPI: found 18 devices
[ 0.637415] ACPI: bus type PNP unregistered
[ 0.643394] pci 0000:00:01.0: PCI bridge to [bus 01]
[ 0.643405] pci 0000:00:02.0: PCI bridge to [bus 02]
[ 0.643408] pci 0000:00:02.0: bridge window [io 0xe000-0xefff]
[ 0.643413] pci 0000:00:02.0: bridge window [mem 0xfb900000-0xfbafffff]
[ 0.643420] pci 0000:00:02.2: PCI bridge to [bus 03]
[ 0.643429] pci 0000:00:03.0: PCI bridge to [bus 04]
[ 0.643439] pci 0000:00:03.1: PCI bridge to [bus 05]
[ 0.643448] pci 0000:00:03.2: PCI bridge to [bus 06]
[ 0.643458] pci 0000:00:11.0: PCI bridge to [bus 07]
[ 0.643461] pci 0000:00:11.0: bridge window [io 0xd000-0xdfff]
[ 0.643470] pci 0000:00:11.0: bridge window [mem 0xfa000000-0xfa4fffff 64bit pref]
[ 0.643477] pci 0000:00:1c.0: PCI bridge to [bus 08]
[ 0.643495] pci 0000:00:1c.6: PCI bridge to [bus 09]
[ 0.643498] pci 0000:00:1c.6: bridge window [io 0xc000-0xcfff]
[ 0.643503] pci 0000:00:1c.6: bridge window [mem 0xfbc00000-0xfbcfffff]
[ 0.643511] pci 0000:00:1c.7: PCI bridge to [bus 0a]
[ 0.643514] pci 0000:00:1c.7: bridge window [io 0xb000-0xbfff]
[ 0.643519] pci 0000:00:1c.7: bridge window [mem 0xfbb00000-0xfbbfffff]
[ 0.643528] pci 0000:00:1e.0: PCI bridge to [bus 0b]
[ 0.643533] pci 0000:00:1e.0: bridge window [mem 0xfb000000-0xfb8fffff]
[ 0.643537] pci 0000:00:1e.0: bridge window [mem 0xf9000000-0xf9ffffff 64bit pref]
[ 0.643543] pci_bus 0000:00: resource 4 [io 0x0000-0x03af]
[ 0.643545] pci_bus 0000:00: resource 5 [io 0x03e0-0x0cf7]
[ 0.643546] pci_bus 0000:00: resource 6 [io 0x03b0-0x03df]
[ 0.643548] pci_bus 0000:00: resource 7 [io 0x0d00-0xffff]
[ 0.643549] pci_bus 0000:00: resource 8 [mem 0x000a0000-0x000bffff]
[ 0.643555] pci_bus 0000:00: resource 9 [mem 0x000c0000-0x000dffff]
[ 0.643557] pci_bus 0000:00: resource 10 [mem 0xfed08000-0xfed08fff]
[ 0.643558] pci_bus 0000:00: resource 11 [mem 0xfed0e000-0xfed0ffff]
[ 0.643560] pci_bus 0000:00: resource 12 [mem 0x80000000-0xfbffffff]
[ 0.643562] pci_bus 0000:02: resource 0 [io 0xe000-0xefff]
[ 0.643563] pci_bus 0000:02: resource 1 [mem 0xfb900000-0xfbafffff]
[ 0.643565] pci_bus 0000:07: resource 0 [io 0xd000-0xdfff]
[ 0.643566] pci_bus 0000:07: resource 2 [mem 0xfa000000-0xfa4fffff 64bit pref]
[ 0.643568] pci_bus 0000:09: resource 0 [io 0xc000-0xcfff]
[ 0.643569] pci_bus 0000:09: resource 1 [mem 0xfbc00000-0xfbcfffff]
[ 0.643571] pci_bus 0000:0a: resource 0 [io 0xb000-0xbfff]
[ 0.643572] pci_bus 0000:0a: resource 1 [mem 0xfbb00000-0xfbbfffff]
[ 0.643574] pci_bus 0000:0b: resource 1 [mem 0xfb000000-0xfb8fffff]
[ 0.643576] pci_bus 0000:0b: resource 2 [mem 0xf9000000-0xf9ffffff 64bit pref]
[ 0.643577] pci_bus 0000:0b: resource 4 [io 0x0000-0x03af]
[ 0.643578] pci_bus 0000:0b: resource 5 [io 0x03e0-0x0cf7]
[ 0.643580] pci_bus 0000:0b: resource 6 [io 0x03b0-0x03df]
[ 0.643581] pci_bus 0000:0b: resource 7 [io 0x0d00-0xffff]
[ 0.643583] pci_bus 0000:0b: resource 8 [mem 0x000a0000-0x000bffff]
[ 0.643584] pci_bus 0000:0b: resource 9 [mem 0x000c0000-0x000dffff]
[ 0.643586] pci_bus 0000:0b: resource 10 [mem 0xfed08000-0xfed08fff]
[ 0.643587] pci_bus 0000:0b: resource 11 [mem 0xfed0e000-0xfed0ffff]
[ 0.643589] pci_bus 0000:0b: resource 12 [mem 0x80000000-0xfbffffff]
[ 0.643616] NET: Registered protocol family 2
[ 0.644007] TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
[ 0.644750] TCP bind hash table entries: 65536 (order: 8, 1048576 bytes)
[ 0.644878] TCP: Hash tables configured (established 524288 bind 65536)
[ 0.644898] TCP: reno registered
[ 0.644961] UDP hash table entries: 32768 (order: 8, 1048576 bytes)
[ 0.645169] UDP-Lite hash table entries: 32768 (order: 8, 1048576 bytes)
[ 0.645373] NET: Registered protocol family 1
[ 0.683698] pci 0000:0b:04.0: Video device with shadowed ROM
[ 0.683743] PCI: CLS 64 bytes, default 64
[ 0.683794] Trying to unpack rootfs image as initramfs...
[ 0.995058] Freeing initrd memory: 20076K (ffff8800358ba000 - ffff880036c55000)
[ 0.995075] PCI-DMA: Using software bounce buffering for IO (SWIOTLB)
[ 0.995078] software IO TLB [mem 0x79d27000-0x7dd27000] (64MB) mapped at [ffff880079d27000-ffff88007dd26fff]
[ 0.995651] microcode: CPU0 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995659] microcode: CPU1 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995669] microcode: CPU2 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995679] microcode: CPU3 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995688] microcode: CPU4 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995698] microcode: CPU5 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995707] microcode: CPU6 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995715] microcode: CPU7 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995723] microcode: CPU8 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995730] microcode: CPU9 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995737] microcode: CPU10 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995746] microcode: CPU11 sig=0x306e4, pf=0x1, revision=0x416
[ 0.995795] microcode: Microcode Update Driver: v2.00 <tigran@aivazian.fsnet.co.uk>, Peter Oruba
[ 0.995799] Scanning for low memory corruption every 60 seconds
[ 0.996076] Initialise system trusted keyring
[ 0.996128] audit: initializing netlink socket (disabled)
[ 0.996139] type=2000 audit(1445682417.828:1): initialized
[ 1.023473] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[ 1.025333] zbud: loaded
[ 1.025459] VFS: Disk quotas dquot_6.5.2
[ 1.025489] Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[ 1.025803] fuse init (API version 7.22)
[ 1.025859] msgmni has been set to 32768
[ 1.025901] Key type big_key registered
[ 1.026273] Key type asymmetric registered
[ 1.026276] Asymmetric key parser 'x509' registered
[ 1.026300] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 252)
[ 1.026350] io scheduler noop registered
[ 1.026353] io scheduler deadline registered (default)
[ 1.026371] io scheduler cfq registered
[ 1.027462] ioapic: probe of 0000:00:05.4 failed with error -22
[ 1.027471] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[ 1.027485] pciehp: PCI Express Hot Plug Controller Driver version: 0.4
[ 1.027524] vesafb: mode is 640x480x32, linelength=2560, pages=0
[ 1.027526] vesafb: scrolling: redraw
[ 1.027528] vesafb: Truecolor: size=8:8:8:8, shift=24:16:8:0
[ 1.042364] vesafb: framebuffer at 0xf9000000, mapped to 0xffffc90016b00000, using 1216k, total 1216k
[ 1.054799] Console: switching to colour frame buffer device 80x30
[ 1.067595] fb0: VESA VGA frame buffer device
[ 1.067871] intel_idle: MWAIT substates: 0x1120
[ 1.067872] intel_idle: v0.4 model 0x3E
[ 1.067873] intel_idle: lapic_timer_reliable_states 0xffffffff
[ 1.068089] ipmi message handler version 39.2
[ 1.068400] input: Power Button as /devices/LNXSYSTM:00/device:00/PNP0C0C:00/input/input0
[ 1.068875] ACPI: Power Button [PWRB]
[ 1.076929] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input1
[ 1.093718] ACPI: Power Button [PWRF]
[ 1.113967] ERST: Error Record Serialization Table (ERST) support is initialized.
[ 1.131776] pstore: Registered erst as persistent store backend
[ 1.140371] GHES: APEI firmware first mode is enabled by WHEA _OSC.
[ 1.151009] Serial: 8250/16550 driver, 32 ports, IRQ sharing enabled
[ 1.181883] 00:09: ttyS0 at I/O 0x3f8 (irq = 4, base_baud = 115200) is a 16550A
[ 1.220231] 00:0a: ttyS1 at I/O 0x2f8 (irq = 3, base_baud = 115200) is a 16550A
[ 1.258702] 00:0c: ttyS2 at I/O 0x3e8 (irq = 10, base_baud = 115200) is a 16550A
[ 1.278405] Linux agpgart interface v0.103
[ 1.289085] brd: module loaded
[ 1.298935] loop: module loaded
[ 1.308648] libphy: Fixed MDIO Bus: probed
[ 1.317959] tun: Universal TUN/TAP device driver, 1.6
[ 1.327098] tun: (C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>
[ 1.336394] PPP generic driver version 2.4.2
[ 1.345595] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver
[ 1.354951] ehci-pci: EHCI PCI platform driver
[ 1.364374] ehci-pci 0000:00:1a.0: EHCI Host Controller
[ 1.373472] ehci-pci 0000:00:1a.0: new USB bus registered, assigned bus number 1
[ 1.391474] ehci-pci 0000:00:1a.0: debug port 2
[ 1.404201] ehci-pci 0000:00:1a.0: cache line size of 64 is not supported
[ 1.404223] ehci-pci 0000:00:1a.0: irq 16, io mem 0xfbd23000
[ 1.424181] ehci-pci 0000:00:1a.0: USB 2.0 started, EHCI 1.00
[ 1.432969] usb usb1: New USB device found, idVendor=1d6b, idProduct=0002
[ 1.441895] usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[ 1.458790] usb usb1: Product: EHCI Host Controller
[ 1.467566] usb usb1: Manufacturer: Linux 3.13.0-66-generic ehci_hcd
[ 1.476162] usb usb1: SerialNumber: 0000:00:1a.0
[ 1.484866] hub 1-0:1.0: USB hub found
[ 1.493146] hub 1-0:1.0: 2 ports detected
[ 1.501441] ehci-pci 0000:00:1d.0: EHCI Host Controller
[ 1.509296] ehci-pci 0000:00:1d.0: new USB bus registered, assigned bus number 2
[ 1.525517] ehci-pci 0000:00:1d.0: debug port 2
[ 1.537274] ehci-pci 0000:00:1d.0: cache line size of 64 is not supported
[ 1.537290] ehci-pci 0000:00:1d.0: irq 23, io mem 0xfbd22000
[ 1.556284] ehci-pci 0000:00:1d.0: USB 2.0 started, EHCI 1.00
[ 1.563912] usb usb2: New USB device found, idVendor=1d6b, idProduct=0002
[ 1.571626] usb usb2: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[ 1.586980] usb usb2: Product: EHCI Host Controller
[ 1.594839] usb usb2: Manufacturer: Linux 3.13.0-66-generic ehci_hcd
[ 1.602918] usb usb2: SerialNumber: 0000:00:1d.0
[ 1.610776] hub 2-0:1.0: USB hub found
[ 1.618410] hub 2-0:1.0: 2 ports detected
[ 1.626206] ehci-platform: EHCI generic platform driver
[ 1.635912] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver
[ 1.645370] ohci-pci: OHCI PCI platform driver
[ 1.655084] ohci-platform: OHCI generic platform driver
[ 1.664017] uhci_hcd: USB Universal Host Controller Interface driver
[ 1.673405] i8042: PNP: PS/2 Controller [PNP0f03:PS2M] at 0x60,0x64 irq 12
[ 1.682764] i8042: PNP: PS/2 controller doesn't have KBD irq; using default 1
[ 1.695037] serio: i8042 KBD port at 0x60,0x64 irq 1
[ 1.704654] serio: i8042 AUX port at 0x60,0x64 irq 12
[ 1.713813] mousedev: PS/2 mouse device common for all mice
[ 1.723276] rtc_cmos 00:03: RTC can wake from S4
[ 1.732284] rtc_cmos 00:03: rtc core: registered rtc_cmos as rtc0
[ 1.741524] rtc_cmos 00:03: alarms up to one month, y3k, 114 bytes nvram, hpet irqs
[ 1.759905] device-mapper: uevent: version 1.0.3
[ 1.769857] device-mapper: ioctl: 4.27.0-ioctl (2013-10-30) initialised: dm-devel@redhat.com
[ 1.789008] ledtrig-cpu: registered to indicate activity on CPUs
[ 1.797305] TCP: cubic registered
[ 1.805425] NET: Registered protocol family 10
[ 1.813585] NET: Registered protocol family 17
[ 1.821683] Key type dns_resolver registered
[ 1.828676] usb 1-1: new high-speed USB device number 2 using ehci-pci
[ 1.837580] Loading compiled-in X.509 certificates
[ 1.846215] Loaded X.509 cert 'Magrathea: Glacier signing key: d957c2fbed0d9fb461efad39c9ccafb5b5e06790'
[ 1.862470] registered taskstats version 1
[ 1.873267] Key type trusted registered
[ 1.884158] Key type encrypted registered
[ 1.892187] AppArmor: AppArmor sha1 policy hashing enabled
[ 1.900043] IMA: No TPM chip found, activating TPM-bypass!
[ 1.908785] regulator-dummy: disabling
[ 1.916828] Magic number: 3:263:427
[ 1.925036] rtc_cmos 00:03: setting system clock to 2015-10-24 10:26:59 UTC (1445682419)
[ 1.942351] BIOS EDD facility v0.16 2004-Jun-25, 0 devices found
[ 1.950357] EDD information not available.
[ 1.958473] PM: Hibernation image not present or could not be loaded.
[ 1.959322] Freeing unused kernel memory: 1336K (ffffffff81d20000 - ffffffff81e6e000)
[ 1.973619] usb 1-1: New USB device found, idVendor=8087, idProduct=0024
[ 1.973621] usb 1-1: New USB device strings: Mfr=0, Product=0, SerialNumber=0
[ 1.974133] hub 1-1:1.0: USB hub found
[ 1.974365] hub 1-1:1.0: 6 ports detected
[ 1.992834] tsc: Refined TSC clocksource calibration: 2399.938 MHz
[ 2.019331] Write protecting the kernel read-only data: 12288k
[ 2.029900] Freeing unused kernel memory: 784K (ffff88000173c000 - ffff880001800000)
[ 2.048443] Freeing unused kernel memory: 680K (ffff880001b56000 - ffff880001c00000)
[ 2.085407] systemd-udevd[177]: starting version 204
[ 2.088825] usb 2-1: new high-speed USB device number 2 using ehci-pci
[ 2.107096] md: linear personality registered for level -1
[ 2.128378] md: multipath personality registered for level -4
[ 2.138278] pps_core: LinuxPPS API ver. 1 registered
[ 2.147354] mpt2sas version 16.100.00.00 loaded
[ 2.147365] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>
[ 2.175390] md: raid0 personality registered for level 0
[ 2.177633] scsi0 : Fusion MPT SAS Host
[ 2.182640] mpt2sas0: 64 BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (65942988 kB)
[ 2.183304] mpt2sas 0000:02:00.0: irq 65 for MSI/MSI-X
[ 2.183375] mpt2sas 0000:02:00.0: irq 66 for MSI/MSI-X
[ 2.183498] mpt2sas 0000:02:00.0: irq 67 for MSI/MSI-X
[ 2.183584] mpt2sas 0000:02:00.0: irq 68 for MSI/MSI-X
[ 2.183695] mpt2sas 0000:02:00.0: irq 69 for MSI/MSI-X
[ 2.183836] mpt2sas 0000:02:00.0: irq 70 for MSI/MSI-X
[ 2.183906] mpt2sas 0000:02:00.0: irq 71 for MSI/MSI-X
[ 2.184070] mpt2sas 0000:02:00.0: irq 72 for MSI/MSI-X
[ 2.184209] mpt2sas 0000:02:00.0: irq 73 for MSI/MSI-X
[ 2.184306] mpt2sas 0000:02:00.0: irq 74 for MSI/MSI-X
[ 2.184367] mpt2sas 0000:02:00.0: irq 75 for MSI/MSI-X
[ 2.184405] mpt2sas 0000:02:00.0: irq 76 for MSI/MSI-X
[ 2.187559] mpt2sas0-msix0: PCI-MSI-X enabled: IRQ 65
[ 2.187561] mpt2sas0-msix1: PCI-MSI-X enabled: IRQ 66
[ 2.187567] mpt2sas0-msix2: PCI-MSI-X enabled: IRQ 67
[ 2.187576] mpt2sas0-msix3: PCI-MSI-X enabled: IRQ 68
[ 2.187578] mpt2sas0-msix4: PCI-MSI-X enabled: IRQ 69
[ 2.187589] mpt2sas0-msix5: PCI-MSI-X enabled: IRQ 70
[ 2.187590] mpt2sas0-msix6: PCI-MSI-X enabled: IRQ 71
[ 2.187591] mpt2sas0-msix7: PCI-MSI-X enabled: IRQ 72
[ 2.187594] mpt2sas0-msix8: PCI-MSI-X enabled: IRQ 73
[ 2.187596] mpt2sas0-msix9: PCI-MSI-X enabled: IRQ 74
[ 2.187597] mpt2sas0-msix10: PCI-MSI-X enabled: IRQ 75
[ 2.187598] mpt2sas0-msix11: PCI-MSI-X enabled: IRQ 76
[ 2.187653] mpt2sas0: iomem(0x00000000fba40000), mapped(0xffffc90006360000), size(65536)
[ 2.187654] mpt2sas0: ioport(0x000000000000e000), size(256)
[ 2.189151] ahci 0000:00:1f.2: version 3.0
[ 2.189731] ahci 0000:00:1f.2: irq 77 for MSI/MSI-X
[ 2.191152] ahci 0000:00:1f.2: AHCI 0001.0300 32 slots 6 ports 6 Gbps 0x3f impl SATA mode
[ 2.191154] ahci 0000:00:1f.2: flags: 64bit ncq sntf pm led clo pio slum part ems apst
[ 2.222915] usb 2-1: New USB device found, idVendor=8087, idProduct=0024
[ 2.222917] usb 2-1: New USB device strings: Mfr=0, Product=0, SerialNumber=0
[ 2.223835] hub 2-1:1.0: USB hub found
[ 2.224132] hub 2-1:1.0: 8 ports detected
[ 2.231413] scsi1 : ahci
[ 2.232122] scsi2 : ahci
[ 2.232547] scsi3 : ahci
[ 2.233610] scsi4 : ahci
[ 2.234092] scsi5 : ahci
[ 2.234662] scsi6 : ahci
[ 2.234869] ata1: SATA max UDMA/133 abar m2048@0xfbd21000 port 0xfbd21100 irq 77
[ 2.234890] ata2: SATA max UDMA/133 abar m2048@0xfbd21000 port 0xfbd21180 irq 77
[ 2.234924] ata3: SATA max UDMA/133 abar m2048@0xfbd21000 port 0xfbd21200 irq 77
[ 2.234947] ata4: SATA max UDMA/133 abar m2048@0xfbd21000 port 0xfbd21280 irq 77
[ 2.234983] ata5: SATA max UDMA/133 abar m2048@0xfbd21000 port 0xfbd21300 irq 77
[ 2.235034] ata6: SATA max UDMA/133 abar m2048@0xfbd21000 port 0xfbd21380 irq 77
[ 2.301126] usb 1-1.6: new full-speed USB device number 3 using ehci-pci
[ 2.396064] usb 1-1.6: New USB device found, idVendor=0557, idProduct=2221
[ 2.396066] usb 1-1.6: New USB device strings: Mfr=1, Product=2, SerialNumber=0
[ 2.396067] usb 1-1.6: Product: Hermon USB hidmouse Device
[ 2.396068] usb 1-1.6: Manufacturer: Winbond Electronics Corp
[ 2.457303] mpt2sas0: sending message unit reset !!
[ 2.469348] mpt2sas0: message unit reset: SUCCESS
[ 2.553622] ata1: SATA link down (SStatus 0 SControl 300)
[ 2.553956] ata3: SATA link down (SStatus 0 SControl 300)
[ 2.557742] ata5: SATA link down (SStatus 0 SControl 300)
[ 2.557984] ata2: SATA link down (SStatus 0 SControl 300)
[ 2.558344] ata4: SATA link down (SStatus 0 SControl 300)
[ 2.558720] ata6: SATA link down (SStatus 0 SControl 300)
[ 2.618404] PTP clock support registered
[ 2.618413] hidraw: raw HID events driver (C) Jiri Kosina
[ 2.636235] isci: Intel(R) C600 SAS Controller Driver - version 1.1.0
[ 2.644773] isci 0000:07:00.0: driver configured for rev: 6 silicon
[ 2.654109] md: raid1 personality registered for level 1
[ 2.654158] e1000e: Intel(R) PRO/1000 Network Driver - 2.3.2-k
[ 2.654158] e1000e: Copyright(c) 1999 - 2013 Intel Corporation.
[ 2.657343] e1000e 0000:09:00.0: Interrupt Throttling Rate (ints/sec) set to dynamic conservative mode
[ 2.658455] e1000e 0000:09:00.0: irq 78 for MSI/MSI-X
[ 2.658483] e1000e 0000:09:00.0: irq 79 for MSI/MSI-X
[ 2.658502] e1000e 0000:09:00.0: irq 80 for MSI/MSI-X
[ 2.663826] usbcore: registered new interface driver usbhid
[ 2.663827] usbhid: USB HID core driver
[ 2.680371] isci 0000:07:00.0: OEM parameter table found in OROM
[ 2.680373] isci 0000:07:00.0: OEM SAS parameters (version: 1.0) loaded (platform)
[ 2.681447] isci 0000:07:00.0: SCU controller 0: phy 3-0 cables: {short, short, short, short}
[ 2.684523] scsi7 : isci
[ 2.685693] isci 0000:07:00.0: irq 81 for MSI/MSI-X
[ 2.685722] isci 0000:07:00.0: irq 82 for MSI/MSI-X
[ 2.693939] mpt2sas0: Allocated physical memory: size(17971 kB)
[ 2.693940] mpt2sas0: Current Controller Queue Depth(9979), Max Controller Queue Depth(10240)
[ 2.693941] mpt2sas0: Scatter Gather Elements per IO(128)
[ 2.779253] e1000e 0000:09:00.0 eth0: registered PHC clock
[ 2.779254] e1000e 0000:09:00.0 eth0: (PCI Express:2.5GT/s:Width x1) 0c:c4:7a:00:bf:a8
[ 2.779255] e1000e 0000:09:00.0 eth0: Intel(R) PRO/1000 Network Connection
[ 2.779418] e1000e 0000:09:00.0 eth0: MAC: 3, PHY: 8, PBA No: FFFFFF-0FF
[ 2.780059] e1000e 0000:0a:00.0: Interrupt Throttling Rate (ints/sec) set to dynamic conservative mode
[ 2.780147] e1000e 0000:0a:00.0: irq 83 for MSI/MSI-X
[ 2.780177] e1000e 0000:0a:00.0: irq 84 for MSI/MSI-X
[ 2.780207] e1000e 0000:0a:00.0: irq 85 for MSI/MSI-X
[ 2.852995] input: Winbond Electronics Corp Hermon USB hidmouse Device as /devices/pci0000:00/0000:00:1a.0/usb1/1-1/1-1.6/1-1.6:1.0/input/input5
[ 2.872074] hid-generic 0003:0557:2221.0001: input,hidraw0: USB HID v1.00 Mouse [Winbond Electronics Corp Hermon USB hidmouse Device] on usb-0000:00:1a.0-1.6/input0
[ 2.899256] e1000e 0000:0a:00.0 eth1: registered PHC clock
[ 2.899257] e1000e 0000:0a:00.0 eth1: (PCI Express:2.5GT/s:Width x1) 0c:c4:7a:00:bf:a9
[ 2.899258] e1000e 0000:0a:00.0 eth1: Intel(R) PRO/1000 Network Connection
[ 2.899425] e1000e 0000:0a:00.0 eth1: MAC: 3, PHY: 8, PBA No: FFFFFF-0FF
[ 2.926767] mpt2sas0: LSISAS2308: FWVersion(17.00.01.00), ChipRevision(0x05), BiosVersion(07.27.00.00)
[ 2.926770] mpt2sas0: Protocol=(Initiator,Target), Capabilities=(TLR,EEDP,Snapshot Buffer,Diag Trace Buffer,Task Set Full,NCQ)
[ 2.927037] mpt2sas0: sending port enable !!
[ 2.934401] mpt2sas0: host_add: handle(0x0001), sas_addr(0x500605b008d85520), phys(8)
[ 2.937476] raid6: sse2x1 6603 MB/s
[ 2.942087] mpt2sas0: expander_add: handle(0x0009), parent(0x0001), sas_addr(0x5003048000e8733f), phys(30)
[ 2.989602] mpt2sas0: port enable: SUCCESS
[ 2.991084] scsi 0:0:0:0: Direct-Access ATA WDC WD1002F9YZ-0 1M02 PQ: 0 ANSI: 6
[ 2.991088] scsi 0:0:0:0: SATA: handle(0x000a), sas_addr(0x5003048000e8732c), phy(12), device_name(0x50014ee0ae74e9a6)
[ 2.991089] scsi 0:0:0:0: SATA: enclosure_logical_id(0x5003048000e8733f), slot(0)
[ 2.991285] scsi 0:0:0:0: atapi(n), ncq(y), asyn_notify(n), smart(y), fua(y), sw_preserve(y)
[ 2.991286] scsi 0:0:0:0: qdepth(32), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1)
[ 2.992897] scsi 0:0:1:0: Direct-Access ATA WDC WD1002F9YZ-0 1M02 PQ: 0 ANSI: 6
[ 2.992901] scsi 0:0:1:0: SATA: handle(0x000b), sas_addr(0x5003048000e8732d), phy(13), device_name(0x50014ee003ca0b80)
[ 2.992902] scsi 0:0:1:0: SATA: enclosure_logical_id(0x5003048000e8733f), slot(1)
[ 2.993101] scsi 0:0:1:0: atapi(n), ncq(y), asyn_notify(n), smart(y), fua(y), sw_preserve(y)
[ 2.993102] scsi 0:0:1:0: qdepth(32), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1)
[ 2.994186] scsi 0:0:2:0: Direct-Access HGST HUS724040ALS640 A1C4 PQ: 0 ANSI: 6
[ 2.994188] scsi 0:0:2:0: SSP: handle(0x000c), sas_addr(0x5000cca03b641e11), phy(14), device_name(0x5000cca03b641e13)
[ 2.994189] scsi 0:0:2:0: SSP: enclosure_logical_id(0x5003048000e8733f), slot(2)
[ 2.994190] scsi 0:0:2:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1)
[ 2.997973] scsi 0:0:3:0: Direct-Access HGST HUS724040ALS640 A1C4 PQ: 0 ANSI: 6
[ 2.997975] scsi 0:0:3:0: SSP: handle(0x000d), sas_addr(0x5000cca03b614c49), phy(15), device_name(0x5000cca03b614c4b)
[ 2.997976] scsi 0:0:3:0: SSP: enclosure_logical_id(0x5003048000e8733f), slot(3)
[ 2.997977] scsi 0:0:3:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1)
[ 2.999685] scsi 0:0:4:0: Direct-Access HGST HUS724040ALS640 A1C4 PQ: 0 ANSI: 6
[ 2.999687] scsi 0:0:4:0: SSP: handle(0x000e), sas_addr(0x5000cca03b649441), phy(16), device_name(0x5000cca03b649443)
[ 2.999688] scsi 0:0:4:0: SSP: enclosure_logical_id(0x5003048000e8733f), slot(4)
[ 2.999689] scsi 0:0:4:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1)
[ 3.005463] raid6: sse2x2 7293 MB/s
[ 3.007580] scsi 0:0:5:0: Direct-Access HGST HUS724040ALS640 A1C4 PQ: 0 ANSI: 6
[ 3.007583] scsi 0:0:5:0: SSP: handle(0x000f), sas_addr(0x5000cca03b632ae1), phy(17), device_name(0x5000cca03b632ae3)
[ 3.007584] scsi 0:0:5:0: SSP: enclosure_logical_id(0x5003048000e8733f), slot(5)
[ 3.007585] scsi 0:0:5:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1)
[ 3.009505] scsi 0:0:6:0: Direct-Access HGST HUS724040ALS640 A1C4 PQ: 0 ANSI: 6
[ 3.009508] scsi 0:0:6:0: SSP: handle(0x0010), sas_addr(0x5000cca03b649399), phy(18), device_name(0x5000cca03b64939b)
[ 3.009509] scsi 0:0:6:0: SSP: enclosure_logical_id(0x5003048000e8733f), slot(6)
[ 3.009511] scsi 0:0:6:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1)
[ 3.014266] scsi 0:0:7:0: Direct-Access HGST HUS724040ALS640 A1C4 PQ: 0 ANSI: 6
[ 3.014268] scsi 0:0:7:0: SSP: handle(0x0011), sas_addr(0x5000cca03b632a81), phy(19), device_name(0x5000cca03b632a83)
[ 3.014269] scsi 0:0:7:0: SSP: enclosure_logical_id(0x5003048000e8733f), slot(7)
[ 3.014270] scsi 0:0:7:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1)
[ 3.016034] scsi 0:0:8:0: Enclosure LSI SAS2X28 0e12 PQ: 0 ANSI: 5
[ 3.016036] scsi 0:0:8:0: SSP: handle(0x0012), sas_addr(0x5003048000e8733d), phy(28), device_name(0x0000000000000000)
[ 3.016037] scsi 0:0:8:0: SSP: enclosure_logical_id(0x5003048000e8733f), slot(0)
[ 3.016038] scsi 0:0:8:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(6), cmd_que(1)
[ 3.073534] raid6: sse2x4 9785 MB/s
[ 3.073535] raid6: using algorithm sse2x4 (9785 MB/s)
[ 3.073535] raid6: using ssse3x2 recovery algorithm
[ 3.974608] Switched to clocksource tsc
[ 3.974732] input: Winbond Electronics Corp Hermon USB hidmouse Device as /devices/pci0000:00/0000:00:1a.0/usb1/1-1/1-1.6/1-1.6:1.1/input/input6
[ 3.974800] hid-generic 0003:0557:2221.0002: input,hidraw1: USB HID v1.00 Keyboard [Winbond Electronics Corp Hermon USB hidmouse Device] on usb-0000:00:1a.0-1.6/input1
[ 4.039690] xor: automatically using best checksumming function:
[ 4.090240] avx : 19898.000 MB/sec
[ 4.101863] async_tx: api initialized (async)
[ 4.117744] md: raid6 personality registered for level 6
[ 4.128047] md: raid5 personality registered for level 5
[ 4.139327] md: raid4 personality registered for level 4
[ 4.153201] md: raid10 personality registered for level 10
[ 4.166394] systemd-udevd[189]: renamed network interface eth1 to em2
[ 4.178467] sd 0:0:0:0: Attached scsi generic sg0 type 0
[ 4.178595] sd 0:0:1:0: Attached scsi generic sg1 type 0
[ 4.178719] sd 0:0:2:0: Attached scsi generic sg2 type 0
[ 4.178816] sd 0:0:3:0: Attached scsi generic sg3 type 0
[ 4.178887] sd 0:0:0:0: [sda] 1953525168 512-byte logical blocks: (1.00 TB/931 GiB)
[ 4.178888] sd 0:0:0:0: [sda] 4096-byte physical blocks
[ 4.178943] sd 0:0:4:0: Attached scsi generic sg4 type 0
[ 4.179030] sd 0:0:1:0: [sdb] 1953525168 512-byte logical blocks: (1.00 TB/931 GiB)
[ 4.179032] sd 0:0:1:0: [sdb] 4096-byte physical blocks
[ 4.179069] sd 0:0:3:0: [sdd] 7814037168 512-byte logical blocks: (4.00 TB/3.63 TiB)
[ 4.179078] sd 0:0:5:0: Attached scsi generic sg5 type 0
[ 4.179192] sd 0:0:6:0: Attached scsi generic sg6 type 0
[ 4.179311] sd 0:0:5:0: [sdf] 7814037168 512-byte logical blocks: (4.00 TB/3.63 TiB)
[ 4.179331] sd 0:0:7:0: Attached scsi generic sg7 type 0
[ 4.179457] scsi 0:0:8:0: Attached scsi generic sg8 type 13
[ 4.179565] sd 0:0:7:0: [sdh] 7814037168 512-byte logical blocks: (4.00 TB/3.63 TiB)
[ 4.179995] sd 0:0:3:0: [sdd] Write Protect is off
[ 4.179997] sd 0:0:3:0: [sdd] Mode Sense: f7 00 10 08
[ 4.180244] sd 0:0:5:0: [sdf] Write Protect is off
[ 4.180246] sd 0:0:5:0: [sdf] Mode Sense: f7 00 10 08
[ 4.180498] sd 0:0:7:0: [sdh] Write Protect is off
[ 4.180499] sd 0:0:7:0: [sdh] Mode Sense: f7 00 10 08
[ 4.180776] sd 0:0:3:0: [sdd] Write cache: enabled, read cache: enabled, supports DPO and FUA
[ 4.181128] sd 0:0:5:0: [sdf] Write cache: enabled, read cache: enabled, supports DPO and FUA
[ 4.181159] sd 0:0:2:0: [sdc] 7814037168 512-byte logical blocks: (4.00 TB/3.63 TiB)
[ 4.181383] sd 0:0:7:0: [sdh] Write cache: enabled, read cache: enabled, supports DPO and FUA
[ 4.182141] sd 0:0:2:0: [sdc] Write Protect is off
[ 4.182142] sd 0:0:2:0: [sdc] Mode Sense: f7 00 10 08
[ 4.182740] sd 0:0:6:0: [sdg] 7814037168 512-byte logical blocks: (4.00 TB/3.63 TiB)
[ 4.183024] sd 0:0:2:0: [sdc] Write cache: enabled, read cache: enabled, supports DPO and FUA
[ 4.183635] sd 0:0:0:0: [sda] Write Protect is off
[ 4.183636] sd 0:0:0:0: [sda] Mode Sense: 7f 00 10 08
[ 4.183727] sd 0:0:6:0: [sdg] Write Protect is off
[ 4.183728] sd 0:0:6:0: [sdg] Mode Sense: f7 00 10 08
[ 4.183816] sd 0:0:1:0: [sdb] Write Protect is off
[ 4.183817] sd 0:0:1:0: [sdb] Mode Sense: 7f 00 10 08
[ 4.184412] sd 0:0:0:0: [sda] Write cache: enabled, read cache: enabled, supports DPO and FUA
[ 4.184578] sd 0:0:1:0: [sdb] Write cache: enabled, read cache: enabled, supports DPO and FUA
[ 4.184664] sd 0:0:6:0: [sdg] Write cache: enabled, read cache: enabled, supports DPO and FUA
[ 4.185579] sd 0:0:4:0: [sde] 7814037168 512-byte logical blocks: (4.00 TB/3.63 TiB)
[ 4.186562] sd 0:0:4:0: [sde] Write Protect is off
[ 4.186563] sd 0:0:4:0: [sde] Mode Sense: f7 00 10 08
[ 4.187449] sd 0:0:4:0: [sde] Write cache: enabled, read cache: enabled, supports DPO and FUA
[ 4.192051] sda: sda1 sda2
[ 4.192205] sdb: sdb1 sdb2
[ 4.199000] sd 0:0:0:0: [sda] Attached SCSI disk
[ 4.199059] sd 0:0:1:0: [sdb] Attached SCSI disk
[ 4.221394] sdh: sdh1 sdh9
[ 4.221528] sdd: sdd1 sdd9
[ 4.227175] sdf: sdf1 sdf9
[ 4.230020] sd 0:0:3:0: [sdd] Attached SCSI disk
[ 4.230166] sd 0:0:7:0: [sdh] Attached SCSI disk
[ 4.231495] sdc: sdc1 sdc9
[ 4.231564] sdg: sdg1 sdg9
[ 4.235697] sd 0:0:5:0: [sdf] Attached SCSI disk
[ 4.237891] sde: sde1 sde9
[ 4.240005] sd 0:0:2:0: [sdc] Attached SCSI disk
[ 4.240079] sd 0:0:6:0: [sdg] Attached SCSI disk
[ 4.246472] sd 0:0:4:0: [sde] Attached SCSI disk
[ 4.282354] random: nonblocking pool is initialized
[ 4.337027] md: bind<sda2>
[ 4.338432] md: bind<sda1>
[ 4.362473] md: bind<sdb1>
[ 4.363639] md/raid1:md0: active with 2 out of 2 mirrors
[ 4.363651] md0: detected capacity change from 0 to 510328832
[ 4.364408] md: bind<sdb2>
[ 4.364921] md0: unknown partition table
[ 4.365608] md/raid1:md1: active with 2 out of 2 mirrors
[ 4.365619] md1: detected capacity change from 0 to 999558021120
[ 4.366715] md1: unknown partition table
[ 4.506545] bio: create slab <bio-1> at 1
[ 4.517478] ses 0:0:8:0: Attached Enclosure device
[ 4.838977] systemd-udevd[190]: renamed network interface eth0 to em1
[ 4.908882] EXT4-fs (dm-1): mounted filesystem with ordered data mode. Opts: (null)
[ 5.692951] zavl: module license 'CDDL' taints kernel.
[ 5.699952] Disabling lock debugging due to kernel taint
[ 5.707217] zavl: module verification failed: signature and/or required key missing - tainting kernel
[ 5.733153] SPL: Loaded module v0.6.5.3-1~trusty
[ 5.797646] ZFS: Loaded module v0.6.5.3-1~trusty, ZFS pool version 5000, ZFS filesystem version 5
[ 7.746782] SPL: using hostid 0x007f0101
[ 13.284828] Adding 5857276k swap on /dev/mapper/nfs01vg0-swapfs. Priority:-1 extents:1 across:5857276k FS
[ 13.351461] EXT4-fs (dm-1): re-mounted. Opts: errors=remount-ro
[ 16.473992] IPv6: ADDRCONF(NETDEV_UP): em1: link is not ready
[ 16.473997] IPv6: ADDRCONF(NETDEV_UP): em2: link is not ready
[ 16.587154] systemd-udevd[2034]: starting version 204
[ 16.666033] FS-Cache: Loaded
[ 16.672414] RPC: Registered named UNIX socket transport module.
[ 16.672416] RPC: Registered udp transport module.
[ 16.672417] RPC: Registered tcp transport module.
[ 16.672417] RPC: Registered tcp NFSv4.1 backchannel transport module.
[ 16.746608] FS-Cache: Netfs 'nfs' registered for caching
[ 16.778740] lp: driver loaded but no devices found
[ 16.822410] bonding: Ethernet Channel Bonding Driver: v3.7.1 (April 27, 2011)
[ 16.865926] Installing knfsd (copyright (C) 1996 okir@monad.swb.de).
[ 17.134636] ACPI Warning: 0x0000000000000428-0x000000000000042f SystemIO conflicts with Region \PMIO 1 (20131115/utaddress-251)
[ 17.134654] ACPI: If an ACPI driver is available for this device, you should use it instead of the native driver
[ 17.134660] ACPI Warning: 0x0000000000000500-0x000000000000052f SystemIO conflicts with Region \GINV 1 (20131115/utaddress-251)
[ 17.134664] ACPI Warning: 0x0000000000000500-0x000000000000052f SystemIO conflicts with Region \GPIO 2 (20131115/utaddress-251)
[ 17.134669] ACPI Warning: 0x0000000000000500-0x000000000000052f SystemIO conflicts with Region \_SB_.PCI0.HEC2.GPO2 3 (20131115/utaddress-251)
[ 17.134673] ACPI: If an ACPI driver is available for this device, you should use it instead of the native driver
[ 17.134702] lpc_ich: Resource conflict(s) found affecting gpio_ich
[ 17.151220] IPMI System Interface driver.
[ 17.151245] ipmi_si: probing via ACPI
[ 17.151268] ipmi_si 00:0d: [io 0x0ca2] regsize 1 spacing 1 irq 0
[ 17.151270] ipmi_si: Adding ACPI-specified kcs state machine
[ 17.151280] ipmi_si: probing via SMBIOS
[ 17.151281] ipmi_si: SMBIOS: io 0xca2 regsize 1 spacing 1 irq 0
[ 17.151282] ipmi_si: Adding SMBIOS-specified kcs state machine duplicate interface
[ 17.151285] ipmi_si: probing via SPMI
[ 17.151286] ipmi_si: SPMI: io 0xca2 regsize 1 spacing 1 irq 0
[ 17.151286] ipmi_si: Adding SPMI-specified kcs state machine duplicate interface
[ 17.151288] ipmi_si: Trying ACPI-specified kcs state machine at i/o address 0xca2, slave address 0x0, irq 0
[ 17.161344] shpchp: Standard Hot Plug PCI Controller Driver version: 0.4
[ 17.177082] mei_me 0000:00:16.0: Device doesn't have valid ME Interface
[ 17.177085] mei_me 0000:00:16.0: initialization failed.
[ 17.188194] wmi: Mapper loaded
[ 17.198501] dca service started, version 1.12.1
[ 17.205272] EDAC MC: Ver: 3.0.0
[ 17.210663] ipmi_si 00:0d: Found new BMC (man_id: 0x002a7c, prod_id: 0x066b, dev_id: 0x20)
[ 17.210674] ipmi_si 00:0d: IPMI kcs interface initialized
[ 17.258893] EDAC sbridge: Seeking for: dev 0e.0 PCI ID 8086:0ea0
[ 17.258903] EDAC sbridge: Seeking for: dev 0e.0 PCI ID 8086:0ea0
[ 17.258905] EDAC sbridge: Seeking for: dev 0f.0 PCI ID 8086:0ea8
[ 17.258909] EDAC sbridge: Seeking for: dev 0f.0 PCI ID 8086:0ea8
[ 17.258911] EDAC sbridge: Seeking for: dev 0f.1 PCI ID 8086:0e71
[ 17.258915] EDAC sbridge: Seeking for: dev 0f.1 PCI ID 8086:0e71
[ 17.258917] EDAC sbridge: Seeking for: dev 0f.2 PCI ID 8086:0eaa
[ 17.258921] EDAC sbridge: Seeking for: dev 0f.2 PCI ID 8086:0eaa
[ 17.258922] EDAC sbridge: Seeking for: dev 0f.3 PCI ID 8086:0eab
[ 17.258926] EDAC sbridge: Seeking for: dev 0f.3 PCI ID 8086:0eab
[ 17.258928] EDAC sbridge: Seeking for: dev 0f.4 PCI ID 8086:0eac
[ 17.258932] EDAC sbridge: Seeking for: dev 0f.4 PCI ID 8086:0eac
[ 17.258933] EDAC sbridge: Seeking for: dev 0f.5 PCI ID 8086:0ead
[ 17.258937] EDAC sbridge: Seeking for: dev 0f.5 PCI ID 8086:0ead
[ 17.258939] EDAC sbridge: Seeking for: dev 16.0 PCI ID 8086:0ec8
[ 17.258943] EDAC sbridge: Seeking for: dev 16.0 PCI ID 8086:0ec8
[ 17.258945] EDAC sbridge: Seeking for: dev 16.1 PCI ID 8086:0ec9
[ 17.258949] EDAC sbridge: Seeking for: dev 16.1 PCI ID 8086:0ec9
[ 17.258950] EDAC sbridge: Seeking for: dev 16.2 PCI ID 8086:0eca
[ 17.258954] EDAC sbridge: Seeking for: dev 16.2 PCI ID 8086:0eca
[ 17.258956] EDAC sbridge: Seeking for: dev 1c.0 PCI ID 8086:0e60
[ 17.258961] EDAC sbridge: Seeking for: dev 1d.2 PCI ID 8086:0e6a
[ 17.258966] EDAC sbridge: Seeking for: dev 1d.3 PCI ID 8086:0e6b
[ 17.258971] EDAC sbridge: Seeking for: dev 11.0 PCI ID 8086:0eb8
[ 17.258975] EDAC sbridge: Seeking for: dev 11.4 PCI ID 8086:0ebc
[ 17.259211] EDAC MC0: Giving out device to module sbridge_edac.c controller Ivy Bridge Socket#0: DEV 0000:ff:0e.0 (POLLED)
[ 17.259213] EDAC sbridge: Driver loaded.
[ 17.286285] ioatdma: Intel(R) QuickData Technology Driver 4.00
[ 17.286458] ioatdma 0000:00:04.0: irq 86 for MSI/MSI-X
[ 17.286713] ioatdma 0000:00:04.1: irq 87 for MSI/MSI-X
[ 17.286927] ioatdma 0000:00:04.2: irq 88 for MSI/MSI-X
[ 17.287136] ioatdma 0000:00:04.3: irq 89 for MSI/MSI-X
[ 17.287346] ioatdma 0000:00:04.4: irq 90 for MSI/MSI-X
[ 17.287563] ioatdma 0000:00:04.5: irq 91 for MSI/MSI-X
[ 17.287771] ioatdma 0000:00:04.6: irq 92 for MSI/MSI-X
[ 17.287978] ioatdma 0000:00:04.7: irq 93 for MSI/MSI-X
[ 17.813753] type=1400 audit(1445682435.372:2): apparmor="STATUS" operation="profile_load" profile="unconfined" name="/sbin/dhclient" pid=2150 comm="apparmor_parser"
[ 17.813759] type=1400 audit(1445682435.372:3): apparmor="STATUS" operation="profile_load" profile="unconfined" name="/usr/lib/NetworkManager/nm-dhcp-client.action" pid=2150 comm="apparmor_parser"
[ 17.813763] type=1400 audit(1445682435.372:4): apparmor="STATUS" operation="profile_load" profile="unconfined" name="/usr/lib/connman/scripts/dhclient-script" pid=2150 comm="apparmor_parser"
[ 17.813774] type=1400 audit(1445682435.372:5): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/sbin/dhclient" pid=2149 comm="apparmor_parser"
[ 17.813780] type=1400 audit(1445682435.372:6): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/NetworkManager/nm-dhcp-client.action" pid=2149 comm="apparmor_parser"
[ 17.813783] type=1400 audit(1445682435.372:7): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/connman/scripts/dhclient-script" pid=2149 comm="apparmor_parser"
[ 17.813793] type=1400 audit(1445682435.372:8): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/sbin/dhclient" pid=2180 comm="apparmor_parser"
[ 17.813799] type=1400 audit(1445682435.372:9): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/NetworkManager/nm-dhcp-client.action" pid=2180 comm="apparmor_parser"
[ 17.813802] type=1400 audit(1445682435.372:10): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/usr/lib/connman/scripts/dhclient-script" pid=2180 comm="apparmor_parser"
[ 17.813814] type=1400 audit(1445682435.372:11): apparmor="STATUS" operation="profile_replace" profile="unconfined" name="/sbin/dhclient" pid=2182 comm="apparmor_parser"
[ 17.954205] init: Failed to obtain startpar-bridge instance: Unknown parameter: INSTANCE
[ 17.959332] bonding: bond0: Setting MII monitoring interval to 100.
[ 18.007496] bonding: bond0: setting mode to active-backup (1).
[ 18.008071] IPv6: ADDRCONF(NETDEV_UP): bond0: link is not ready
[ 18.061261] bonding: bond0: Adding slave em2.
[ 18.139906] IPv6: ADDRCONF(NETDEV_UP): em2: link is not ready
[ 18.139936] bonding: bond0: enslaving em2 as a backup interface with a down link.
[ 18.139939] bonding: bond0: Adding slave em1.
[ 18.219984] IPv6: ADDRCONF(NETDEV_UP): em1: link is not ready
[ 18.220011] bonding: bond0: enslaving em1 as a backup interface with a down link.
[ 18.222386] bonding: bond0: Setting em1 as primary slave.
[ 18.362267] init: failsafe main process (2338) killed by TERM signal
[ 18.938130] ip_tables: (C) 2000-2006 Netfilter Core Team
[ 18.958171] nf_conntrack version 0.5.0 (16384 buckets, 65536 max)
[ 19.007544] ip6_tables: (C) 2000-2006 Netfilter Core Team
[ 19.284840] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory
[ 19.284935] NFSD: starting 90-second grace period (net ffffffff81cdab80)
[ 21.236374] e1000e: em2 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: None
[ 21.311381] bonding: bond0: link status definitely up for interface em2, 1000 Mbps full duplex.
[ 21.311385] bonding: bond0: making interface em2 the new active one.
[ 21.311838] bonding: bond0: first active interface up!
[ 21.311842] IPv6: ADDRCONF(NETDEV_CHANGE): bond0: link becomes ready
[ 21.328404] e1000e: em1 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: None
[ 21.411429] bonding: bond0: link status definitely up for interface em1, 1000 Mbps full duplex.
[ 21.411432] bonding: bond0: making interface em1 the new active one.
[ 22.378513] init: plymouth-upstart-bridge main process ended, respawning
[12148.160880] perf samples too long (2514 > 2500), lowering kernel.perf_event_max_sample_rate to 50000
[42497.144462] perf samples too long (5063 > 5000), lowering kernel.perf_event_max_sample_rate to 25000
[131868.047227] INFO: task sh:32547 blocked for more than 120 seconds.
[131868.047586] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[131868.047971] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[131868.048433] sh D ffff88107fd53180 0 32547 32540 0x00000000
[131868.048437] ffff880dde089c50 0000000000000086 ffff8810284ec800 ffff880dde089fd8
[131868.048442] 0000000000013180 0000000000013180 ffff8810284ec800 ffff881022e64268
[131868.048445] ffff881022e6426c ffff8810284ec800 00000000ffffffff ffff881022e64270
[131868.048448] Call Trace:
[131868.048456] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[131868.048460] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[131868.048463] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[131868.048467] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[131868.048471] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[131868.048475] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[131868.048480] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[131868.048483] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[131868.048487] [<ffffffff810777ab>] ? recalc_sigpending+0x1b/0x50
[131868.048490] [<ffffffff811ce32f>] ? getname_flags+0x4f/0x190
[131868.048494] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[131868.048497] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[131868.048502] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[131868.048505] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[131868.048508] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[131988.144703] INFO: task sh:32547 blocked for more than 120 seconds.
[131988.145056] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[131988.145440] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[131988.145902] sh D ffff88107fd53180 0 32547 32540 0x00000000
[131988.145905] ffff880dde089c50 0000000000000086 ffff8810284ec800 ffff880dde089fd8
[131988.145910] 0000000000013180 0000000000013180 ffff8810284ec800 ffff881022e64268
[131988.145913] ffff881022e6426c ffff8810284ec800 00000000ffffffff ffff881022e64270
[131988.145917] Call Trace:
[131988.145924] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[131988.145928] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[131988.145930] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[131988.145935] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[131988.145938] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[131988.145942] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[131988.145947] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[131988.145950] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[131988.145954] [<ffffffff810777ab>] ? recalc_sigpending+0x1b/0x50
[131988.145957] [<ffffffff811ce32f>] ? getname_flags+0x4f/0x190
[131988.145961] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[131988.145964] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[131988.145969] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[131988.145972] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[131988.145975] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[131988.145979] INFO: task sh:648 blocked for more than 120 seconds.
[131988.146320] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[131988.146705] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[131988.147262] sh D ffff88107fd53180 0 648 647 0x00000000
[131988.147265] ffff880ae17c5c50 0000000000000082 ffff881027534800 ffff880ae17c5fd8
[131988.147269] 0000000000013180 0000000000013180 ffff881027534800 ffff881022e64268
[131988.147272] ffff881022e6426c ffff881027534800 00000000ffffffff ffff881022e64270
[131988.147275] Call Trace:
[131988.147278] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[131988.147281] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[131988.147283] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[131988.147286] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[131988.147289] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[131988.147292] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[131988.147295] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[131988.147298] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[131988.147302] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[131988.147304] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[131988.147307] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[131988.147310] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[131988.147313] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[131988.147315] INFO: task sh:740 blocked for more than 120 seconds.
[131988.147716] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[131988.148105] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[131988.148572] sh D ffff88107fd33180 0 740 739 0x00000000
[131988.148574] ffff880b71fd9c50 0000000000000082 ffff880e763dc800 ffff880b71fd9fd8
[131988.148578] 0000000000013180 0000000000013180 ffff880e763dc800 ffff881022e64268
[131988.148581] ffff881022e6426c ffff880e763dc800 00000000ffffffff ffff881022e64270
[131988.148584] Call Trace:
[131988.148587] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[131988.148589] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[131988.148592] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[131988.148594] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[131988.148597] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[131988.148600] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[131988.148603] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[131988.148606] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[131988.148609] [<ffffffff810777ab>] ? recalc_sigpending+0x1b/0x50
[131988.148612] [<ffffffff8107aa46>] ? __set_current_blocked+0x36/0x80
[131988.148615] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[131988.148618] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[131988.148621] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[131988.148624] [<ffffffff8107b919>] ? restore_altstack+0x19/0x30
[131988.148629] [<ffffffff81014096>] ? sys_rt_sigreturn+0xb6/0xc0
[131988.148632] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[131988.148642] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[132108.246183] INFO: task sh:32547 blocked for more than 120 seconds.
[132108.246537] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[132108.246923] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[132108.247384] sh D ffff88107fd53180 0 32547 32540 0x00000000
[132108.247388] ffff880dde089c50 0000000000000086 ffff8810284ec800 ffff880dde089fd8
[132108.247392] 0000000000013180 0000000000013180 ffff8810284ec800 ffff881022e64268
[132108.247395] ffff881022e6426c ffff8810284ec800 00000000ffffffff ffff881022e64270
[132108.247399] Call Trace:
[132108.247407] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[132108.247411] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[132108.247413] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[132108.247418] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[132108.247421] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[132108.247425] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[132108.247430] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[132108.247433] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[132108.247437] [<ffffffff810777ab>] ? recalc_sigpending+0x1b/0x50
[132108.247440] [<ffffffff811ce32f>] ? getname_flags+0x4f/0x190
[132108.247444] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[132108.247447] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[132108.247451] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[132108.247454] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[132108.247458] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[132108.247461] INFO: task sh:648 blocked for more than 120 seconds.
[132108.247800] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[132108.248188] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[132108.248648] sh D ffff88107fd53180 0 648 647 0x00000000
[132108.248651] ffff880ae17c5c50 0000000000000082 ffff881027534800 ffff880ae17c5fd8
[132108.248654] 0000000000013180 0000000000013180 ffff881027534800 ffff881022e64268
[132108.248657] ffff881022e6426c ffff881027534800 00000000ffffffff ffff881022e64270
[132108.248661] Call Trace:
[132108.248664] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[132108.248666] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[132108.248669] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[132108.248672] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[132108.248675] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[132108.248678] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[132108.248681] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[132108.248684] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[132108.248687] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[132108.248690] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[132108.248693] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[132108.248695] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[132108.248698] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[132108.248700] INFO: task sh:740 blocked for more than 120 seconds.
[132108.249039] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[132108.259284] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[132108.280357] sh D ffff88107fd33180 0 740 739 0x00000000
[132108.280360] ffff880b71fd9c50 0000000000000082 ffff880e763dc800 ffff880b71fd9fd8
[132108.280364] 0000000000013180 0000000000013180 ffff880e763dc800 ffff881022e64268
[132108.280367] ffff881022e6426c ffff880e763dc800 00000000ffffffff ffff881022e64270
[132108.280370] Call Trace:
[132108.280374] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[132108.280377] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[132108.280379] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[132108.280382] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[132108.280385] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[132108.280388] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[132108.280391] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[132108.280394] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[132108.280397] [<ffffffff810777ab>] ? recalc_sigpending+0x1b/0x50
[132108.280401] [<ffffffff8107aa46>] ? __set_current_blocked+0x36/0x80
[132108.280404] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[132108.280406] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[132108.280410] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[132108.280413] [<ffffffff8107b919>] ? restore_altstack+0x19/0x30
[132108.280418] [<ffffffff81014096>] ? sys_rt_sigreturn+0xb6/0xc0
[132108.280421] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[132108.280424] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[132108.280426] INFO: task sh:837 blocked for more than 120 seconds.
[132108.291041] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[132108.301873] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[132108.323202] sh D ffff88107fc13180 0 837 836 0x00000000
[132108.323205] ffff880de1a35c50 0000000000000086 ffff881018d58000 ffff880de1a35fd8
[132108.323209] 0000000000013180 0000000000013180 ffff881018d58000 ffff881022e64268
[132108.323212] ffff881022e6426c ffff881018d58000 00000000ffffffff ffff881022e64270
[132108.323216] Call Trace:
[132108.323219] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[132108.323222] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[132108.323225] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[132108.323228] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[132108.323231] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[132108.323234] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[132108.323237] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[132108.323241] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[132108.323244] [<ffffffff810777ab>] ? recalc_sigpending+0x1b/0x50
[132108.323247] [<ffffffff8107aa46>] ? __set_current_blocked+0x36/0x80
[132108.323250] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[132108.323253] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[132108.323256] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[132108.323259] [<ffffffff8107b919>] ? restore_altstack+0x19/0x30
[132108.323263] [<ffffffff81014096>] ? sys_rt_sigreturn+0xb6/0xc0
[132108.323266] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[132108.323268] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[132108.323271] INFO: task sh:933 blocked for more than 120 seconds.
[132108.333894] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[132108.344464] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[132108.365702] sh D ffff88107fcb3180 0 933 932 0x00000000
[132108.365707] ffff88001f6b9c50 0000000000000082 ffff88101aa4b000 ffff88001f6b9fd8
[132108.365711] 0000000000013180 0000000000013180 ffff88101aa4b000 ffff881022e64268
[132108.365717] ffff881022e6426c ffff88101aa4b000 00000000ffffffff ffff881022e64270
[132108.365721] Call Trace:
[132108.365727] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[132108.365730] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[132108.365733] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[132108.365736] [<ffffffff811ccca5>] do_last+0x2a5/0x1240
[132108.365739] [<ffffffff811cac06>] ? link_path_walk+0x256/0x880
[132108.365742] [<ffffffff8131615b>] ? apparmor_file_alloc_security+0x5b/0x180
[132108.365745] [<ffffffff812d8786>] ? security_file_alloc+0x16/0x20
[132108.365748] [<ffffffff811cdcfb>] path_openat+0xbb/0x650
[132108.365752] [<ffffffff810c949d>] ? call_rcu_sched+0x1d/0x20
[132108.365756] [<ffffffff8109e6b1>] ? thread_group_cputime_adjusted+0x41/0x50
[132108.365759] [<ffffffff811cf0fa>] do_filp_open+0x3a/0x90
[132108.365762] [<ffffffff811dbf77>] ? __alloc_fd+0xa7/0x130
[132108.365765] [<ffffffff811bd6d9>] do_sys_open+0x129/0x280
[132108.365767] [<ffffffff811bd84e>] SyS_open+0x1e/0x20
[132108.365770] [<ffffffff81734c5d>] system_call_fastpath+0x1a/0x1f
[148201.438297] INFO: task nfsd:3043 blocked for more than 120 seconds.
[148201.451037] Tainted: P OX 3.13.0-66-generic #108-Ubuntu
[148201.463751] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[148201.487483] nfsd D ffff88107fcd3180 0 3043 2 0x00000000
[148201.487488] ffff88100928fad0 0000000000000046 ffff88100ac3e000 ffff88100928ffd8
[148201.487492] 0000000000013180 0000000000013180 ffff88100ac3e000 ffff880b248ce028
[148201.487495] ffff880b248ce02c ffff88100ac3e000 00000000ffffffff ffff880b248ce030
[148201.487498] Call Trace:
[148201.487506] [<ffffffff817289b9>] schedule_preempt_disabled+0x29/0x70
[148201.487510] [<ffffffff8172a825>] __mutex_lock_slowpath+0x135/0x1b0
[148201.487513] [<ffffffff8172a8bf>] mutex_lock+0x1f/0x2f
[148201.487517] [<ffffffff812aca85>] reconnect_path+0x95/0x2d0
[148201.487531] [<ffffffffa060a8d0>] ? nfsd_proc_getattr+0xa0/0xa0 [nfsd]
[148201.487536] [<ffffffff812ace09>] exportfs_decode_fh+0xf9/0x2e0
[148201.487540] [<ffffffff8109aca2>] ? default_wake_function+0x12/0x20
[148201.487544] [<ffffffff810ab3a2>] ? autoremove_wake_function+0x12/0x40
[148201.487547] [<ffffffff810aaca8>] ? __wake_up_common+0x58/0x90
[148201.487551] [<ffffffff810aaf34>] ? __wake_up+0x44/0x50
[148201.487554] [<ffffffff810c8568>] ? __call_rcu_nocb_enqueue+0xa8/0xc0
[148201.487560] [<ffffffffa060b5f6>] fh_verify+0x2f6/0x5e0 [nfsd]
[148201.487575] [<ffffffffa0523a30>] ? svcauth_unix_domain_release+0x30/0x30 [sunrpc]
[148201.487578] [<ffffffff810cd340>] ? getboottime+0x30/0x40
[148201.487585] [<ffffffffa060f57d>] nfsd_access+0x2d/0x130 [nfsd]
[148201.487595] [<ffffffffa0524c05>] ? svcauth_unix_set_client+0x4c5/0x580 [sunrpc]
[148201.487603] [<ffffffffa0615911>] nfsd3_proc_access+0x71/0xc0 [nfsd]
[148201.487609] [<ffffffffa0607d3b>] nfsd_dispatch+0xbb/0x200 [nfsd]
[148201.487618] [<ffffffffa052063d>] svc_process_common+0x46d/0x6d0 [sunrpc]
[148201.487627] [<ffffffffa05209a7>] svc_process+0x107/0x170 [sunrpc]
[148201.487633] [<ffffffffa060771f>] nfsd+0xbf/0x130 [nfsd]
[148201.487638] [<ffffffffa0607660>] ? nfsd_destroy+0x80/0x80 [nfsd]
[148201.487642] [<ffffffff8108b7d2>] kthread+0xd2/0xf0
[148201.487645] [<ffffffff8108b700>] ? kthread_create_on_node+0x1c0/0x1c0
[148201.487648] [<ffffffff81734ba8>] ret_from_fork+0x58/0x90
[148201.487651] [<ffffffff8108b700>] ? kthread_create_on_node+0x1c0/0x1c0# top -n -c1
top - 07:44:50 up 1 day, 20:17, 1 user, load average: 27.18, 25.72, 19.15
Tasks: 669 total, 11 running, 658 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0.2 us, 3.8 sy, 0.0 ni, 92.6 id, 3.3 wa, 0.0 hi, 0.2 si, 0.0 st
KiB Mem: 65942988 total, 35771312 used, 30171676 free, 254148 buffers
KiB Swap: 5857276 total, 0 used, 5857276 free. 416888 cached Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
3044 root 20 0 0 0 0 R 93.6 0.0 60:36.01 [nfsd]
3048 root 20 0 0 0 0 R 93.6 0.0 46:14.26 [nfsd]
3049 root 20 0 0 0 0 R 93.6 0.0 30:34.31 [nfsd]
3033 root 20 0 0 0 0 R 87.8 0.0 42:07.90 [nfsd]
3038 root 20 0 0 0 0 R 87.8 0.0 57:44.76 [nfsd]
3050 root 20 0 0 0 0 R 87.8 0.0 53:28.82 [nfsd]
3051 root 20 0 0 0 0 R 87.8 0.0 50:25.77 [nfsd]
3039 root 20 0 0 0 0 R 81.9 0.0 36:59.70 [nfsd]
3045 root 20 0 0 0 0 R 81.9 0.0 42:06.12 [nfsd]
3046 root 20 0 0 0 0 R 41.0 0.0 41:55.25 [nfsd]
473 root 20 0 0 0 0 D 17.6 0.0 2:22.06 [arc_prune]
26078 root 20 0 22512 1976 1028 R 11.7 0.0 0:00.03 top -c -n 1
1408 root 0 -20 0 0 0 S 5.9 0.0 0:11.10 [z_rd_int_2]
1473 root 0 -20 0 0 0 S 5.9 0.0 0:11.10 [z_rd_int_7]
3042 root 20 0 0 0 0 D 5.9 0.0 35:54.10 [nfsd]
1 root 20 0 33604 2912 1472 S 0.0 0.0 0:02.61 /sbin/init
2 root 20 0 0 0 0 S 0.0 0.0 0:00.05 [kthreadd]
3 root 20 0 0 0 0 S 0.0 0.0 0:00.73 [ksoftirqd/0]
5 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 [kworker/0:0H]
7 root 20 0 0 0 0 S 0.0 0.0 5:28.15 [rcu_sched]
8 root 20 0 0 0 0 S 0.0 0.0 0:37.43 [rcuos/0]
9 root 20 0 0 0 0 S 0.0 0.0 10:14.42 [rcuos/1]
10 root 20 0 0 0 0 S 0.0 0.0 3:51.81 [rcuos/2]
11 root 20 0 0 0 0 S 0.0 0.0 1:36.24 [rcuos/3]
12 root 20 0 0 0 0 S 0.0 0.0 0:58.54 [rcuos/4]
13 root 20 0 0 0 0 S 0.0 0.0 0:40.12 [rcuos/5]
14 root 20 0 0 0 0 S 0.0 0.0 0:10.51 [rcuos/6]
15 root 20 0 0 0 0 S 0.0 0.0 0:09.54 [rcuos/7]
16 root 20 0 0 0 0 S 0.0 0.0 0:10.91 [rcuos/8]
17 root 20 0 0 0 0 S 0.0 0.0 0:10.20 [rcuos/9]
18 root 20 0 0 0 0 S 0.0 0.0 0:09.86 [rcuos/10]
19 root 20 0 0 0 0 S 0.0 0.0 0:10.26 [rcuos/11]
20 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [rcu_bh]
21 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [rcuob/0]
22 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [rcuob/1]
23 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [rcuob/2]
24 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [rcuob/3]
25 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [rcuob/4]
26 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [rcuob/5]
27 root 20 0 0 0 0 S 0.0 0.0 0:00.00 [rcuob/6] all these logs and outputs were taken while the error occured. Is ist possible that this bug is related to this issue?
OverLocker
hjjg
tomposmiko
dmesg is totally empty, there are no those sort of "stuck for 120 sec" messages. However as you can see, txg_sync and other kernel threads eats up all CPU and zfs/zpool userland tools are stuck.