search

alpinelinux / docker-alpine

Official Alpine Linux Docker image. Win at minimalism!
MIT License
1.11k stars 264 forks source link

run alpine and cpu lockup #119

Open lurenjia528 opened 4 years ago

lurenjia528 commented 4 years ago

env:

root@node-4:~# uname -a 
Linux node-4 4.4.131-20190726.kylin.server-generic #kylin SMP Tue Jul 30 16:44:09 CST 2019 aarch64 aarch64 aarch64 GNU/Linux

root@node-4:~# docker version
Client: Docker Engine - Community
 Version:           19.03.9
 API version:       1.40
 Go version:        go1.13.10
 Git commit:        9d98839
 Built:             Fri May 15 00:25:37 2020
 OS/Arch:           linux/arm64
 Experimental:      false

Server: Docker Engine - Community
 Engine:
  Version:          19.03.9
  API version:      1.40 (minimum version 1.12)
  Go version:       go1.13.10
  Git commit:       9d98839
  Built:            Fri May 15 00:24:11 2020
  OS/Arch:          linux/arm64
  Experimental:     true
 containerd:
  Version:          1.3.7
  GitCommit:        8fba4e9a7d01810a393d5d25a3621dc101981175
 runc:
  Version:          1.0.0-rc10
  GitCommit:        dc9208a3303feef5b3839f4323d9beb36df0a9dd
 docker-init:
  Version:          0.18.0
  GitCommit:        fec3683

run the command docker run --rm -it arm64v8/alpine:3.12 sh and then ,

0,13055,786664246336,-;NMI watchdog: BUG: soft lockup - CPU#34 stuck for 23s! [sh:42014]
4,13056,786664253527,c;Modules linked in: xt_REDIRECT nf_nat_redirect xt_tcpudp xt_owner xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth ip_set_bitmap_port ip_set_hash_ipportnet ip_set_hash_ipport ip_set_hash_ipportip ip_set dummy ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle xt_comment xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xfrm_user xfrm_algo xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ip_tables x_tables br_netfilter bridge stp llc aufs rpcsec_gss_krb5 auth_rpcgss nfsv4 nfs lockd grace fscache overlay joydev igb ixgbe vxlan ip6_udp_tunnel udp_tunnel mdio shpchp dms5013a crc32_arm64 sunrpc autofs4
4,13057,786664253592,+; megaraid_sas
4,13058,786664253595,-;
4,13059,786664253599,-;CPU: 34 PID: 42014 Comm: sh Tainted: G             L  4.4.131-20190726.kylin.server-generic #kylin
4,13060,786664253602,-;Hardware name: GreatWall \xe6\x5cx93\x5cx8e\xe5\xa4\xa9DF720/N/A, BIOS 601FBE23 Dec  3 2019
4,13061,786664253604,-;task: ffff80835f0ae140 ti: ffff808350c48000 task.ti: ffff808350c48000
4,13062,786664253611,-;PC is at _raw_spin_unlock_irqrestore+0x2c/0x38
4,13063,786664253616,-;LR is at __wake_up+0x58/0x68
4,13064,786664253618,-;pc : [<ffff800000a150fc>] lr : [<ffff800000107018>] pstate: 60000145
4,13065,786664253619,-;sp : ffff808350c4bc00
4,13066,786664253621,-;x29: ffff808350c4bc00 x28: ffff84834e2e0800 
4,13067,786664253624,-;x27: ffff8000004db870 x26: ffff808350c48000 
4,13068,786664253627,-;x25: 0000000000000000 x24: 0000000000000000 
4,13069,786664253630,-;x23: ffff84834864b0c0 x22: 0000000000000004 
4,13070,786664253633,-;x21: 0000000000000001 x20: ffff84834e2e0a30 
4,13071,786664253635,-;x19: 0000000000000140 x18: 2000000000000000 
4,13072,786664253638,-;x17: 0000ffff7921d540 x16: ffff80000024ced8 
4,13073,786664253641,-;x15: 000000000d39ad3d x14: 0000ffff79277f50 
4,13074,786664253643,-;x13: 0000000000000018 x12: 000000000d39ad3c 
4,13075,786664253646,-;x11: 00000000ffffffff x10: ffff808350c48000 
4,13076,786664253649,-;x9 : 000000007fff0000 x8 : 0001000000000000 
4,13077,786664253651,-;x7 : 0000000000000001 x6 : ffff84834d289400 
4,13078,786664253654,-;x5 : dead000000000100 x4 : 0000000000000004 
4,13079,786664253656,-;x3 : 0000000000000000 x2 : 0000000000000001 
4,13080,786664253659,-;x1 : 0000000000000140 x0 : 0000000000007ac6 
4,13081,786664253661,-;
0,13082,786692246335,-;NMI watchdog: BUG: soft lockup - CPU#34 stuck for 23s! [sh:42014]
4,13083,786692253523,c;Modules linked in: xt_REDIRECT nf_nat_redirect xt_tcpudp xt_owner xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth ip_set_bitmap_port ip_set_hash_ipportnet ip_set_hash_ipport ip_set_hash_ipportip ip_set dummy ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle xt_comment xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xfrm_user xfrm_algo xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ip_tables x_tables br_netfilter bridge stp llc aufs rpcsec_gss_krb5 auth_rpcgss nfsv4 nfs lockd grace fscache overlay joydev igb ixgbe vxlan ip6_udp_tunnel udp_tunnel mdio shpchp dms5013a crc32_arm64 sunrpc autofs4
4,13084,786692253589,+; megaraid_sas
4,13085,786692253591,-;
4,13086,786692253596,-;CPU: 34 PID: 42014 Comm: sh Tainted: G             L  4.4.131-20190726.kylin.server-generic #kylin
4,13087,786692253598,-;Hardware name: GreatWall \xe6\x5cx93\x5cx8e\xe5\xa4\xa9DF720/N/A, BIOS 601FBE23 Dec  3 2019
4,13088,786692253601,-;task: ffff80835f0ae140 ti: ffff808350c48000 task.ti: ffff808350c48000
4,13089,786692253608,-;PC is at _raw_spin_unlock_irqrestore+0x2c/0x38
4,13090,786692253613,-;LR is at add_wait_queue+0x58/0x68
4,13091,786692253615,-;pc : [<ffff800000a150fc>] lr : [<ffff800000106e48>] pstate: 60000145
4,13092,786692253617,-;sp : ffff808350c4bb80
4,13093,786692253619,-;x29: ffff808350c4bb80 x28: ffff84834e2e0800 
4,13094,786692253622,-;x27: ffff8000004db870 x26: ffff84834e2e08d8 
4,13095,786692253625,-;x25: ffff878347729300 x24: 0000000000000000 
4,13096,786692253627,-;x23: ffff84834864b0c0 x22: ffff878347729300 
4,13097,786692253630,-;x21: ffff808350c4bc60 x20: ffff84834e2e0a30 
4,13098,786692253633,-;x19: 0000000000000140 x18: 2000000000000000 
4,13099,786692253636,-;x17: 0000ffff7921d540 x16: ffff80000024ced8 
4,13100,786692253638,-;x15: 000000000d39ad3d x14: 0000ffff79277f50 
4,13101,786692253641,-;x13: 0000000000000018 x12: 000000000d39ad3c 
4,13102,786692253644,-;x11: 00000000ffffffff x10: ffff808350c48000 
4,13103,786692253646,-;x9 : 000000007fff0000 x8 : 0001000000000000 
4,13104,786692253649,-;x7 : 0000000000000001 x6 : ffff84834d289400 
4,13105,786692253651,-;x5 : ffff84834d289400 x4 : 0001000000000000 
4,13106,786692253654,-;x3 : ffff84834e2e0a38 x2 : ffff84834e2e0a38 
4,13107,786692253657,-;x1 : 0000000000000140 x0 : 000000000000673b 
4,13108,786692253659,-;
3,13109,786699996336,-;INFO: rcu_sched self-detected stall on CPU
3,13110,786700001552,-;\x0934-...: (5999 ticks this GP) idle=893/140000000000001/0 softirq=13250105/13250105 fqs=5775 
3,13111,786700011070,-;\x09 (t=6001 jiffies g=36719561 c=36719560 q=83188)
3,13112,786700016328,-;INFO: rcu_sched detected stalls on CPUs/tasks:
3,13113,786700016336,-;\x0934-...: (5999 ticks this GP) idle=893/140000000000001/0 softirq=13250105/13250105 fqs=5775 
3,13114,786700016386,-;\x09(detected by 7, t=6002 jiffies, g=36719561, c=36719560, q=83188)
6,13115,786700016387,-;Task dump for CPU 34:
6,13116,786700016391,-;sh              R  running task        0 42014  41994 0x00000806
0,13117,786700016392,-;Call trace:
4,13118,786700016398,-;[<ffff8000000861c4>] __switch_to+0xa4/0xb8
4,13119,786700016400,-;[<0002cb7fdb10b980>] 0x2cb7fdb10b980
6,13120,786700041564,-;Task dump for CPU 34:
6,13121,786700041566,-;sh              R  running task        0 42014  41994 0x00000806
0,13122,786700041570,-;Call trace:
4,13123,786700044091,-;[<ffff8000000897f8>] dump_backtrace+0x0/0x178
4,13124,786700044094,-;[<ffff800000089b1c>] show_stack+0x24/0x30
4,13125,786700044099,-;[<ffff8000001cbcb4>] sched_show_task+0xd0/0xdc
4,13126,786700044102,-;[<ffff8000001cc2c8>] dump_cpu_task+0x48/0x54
4,13127,786700044104,-;[<ffff8000001cce48>] rcu_dump_cpu_stacks+0x98/0xb4
4,13128,786700044107,-;[<ffff8000001280d8>] rcu_check_callbacks+0x758/0x8b8
4,13129,786700044110,-;[<ffff80000012da8c>] update_process_times+0x44/0x90
4,13130,786700044114,-;[<ffff80000013ec90>] tick_sched_handle.isra.0+0x38/0x78
4,13131,786700044116,-;[<ffff80000013ed1c>] tick_sched_timer+0x4c/0x90
4,13132,786700044119,-;[<ffff80000012e458>] __hrtimer_run_queues+0x148/0x2c0
4,13133,786700044121,-;[<ffff80000012ecc0>] hrtimer_interrupt+0xa0/0x1d0
4,13134,786700044126,-;[<ffff80000089ba24>] arch_timer_handler_phys+0x3c/0x50
4,13135,786700044128,-;[<ffff80000011b85c>] handle_percpu_devid_irq+0x9c/0x228
4,13136,786700044132,-;[<ffff800000116d1c>] generic_handle_irq+0x34/0x50
4,13137,786700044134,-;[<ffff80000011706c>] __handle_domain_irq+0x6c/0xc0
4,13138,786700044137,-;[<ffff800000081c88>] gic_handle_irq+0x98/0x194
4,13139,786700044139,-;Exception stack(0xffff808350c4baa0 to 0xffff808350c4bbc0)
4,13140,786700044142,-;baa0: 0000000000000140 ffff84834e2e0a30 ffff808350c4bc00 ffff800000a150fc
4,13141,786700044145,-;bac0: 0000000060000145 0000000000000000 0000000000000000 ffff808350c48000
4,13142,786700044147,-;bae0: 000000000000d0d8 0000000000000140 0000000000000001 0000000000000000
4,13143,786700044150,-;bb00: 0000000000000004 dead000000000100 ffff84834d289400 0000000000000001
4,13144,786700044152,-;bb20: 0001000000000000 000000007fff0000 ffff808350c48000 00000000ffffffff
4,13145,786700044154,-;bb40: 000000000d39ad3c 0000000000000018 0000ffff79277f50 000000000d39ad3d
4,13146,786700044157,-;bb60: ffff80000024ced8 0000ffff7921d540 2000000000000000 0000000000000140
4,13147,786700044159,-;bb80: ffff84834e2e0a30 0000000000000001 0000000000000004 ffff84834864b0c0
4,13148,786700044161,-;bba0: 0000000000000000 0000000000000000 ffff808350c48000 ffff8000004db870
4,13149,786700044164,-;[<ffff800000084da8>] el1_irq+0x68/0xc0
4,13150,786700044167,-;[<ffff800000107018>] __wake_up+0x58/0x68
4,13151,786700044172,-;[<ffff8000004d5404>] tty_write_unlock+0x34/0x40
4,13152,786700044174,-;[<ffff8000004d7910>] tty_write+0x190/0x268
4,13153,786700044178,-;[<ffff80000024af04>] do_loop_readv_writev.part.0+0x84/0xb8
4,13154,786700044180,-;[<ffff80000024b7ac>] do_readv_writev+0x1dc/0x270
4,13155,786700044183,-;[<ffff80000024bb18>] vfs_writev+0x58/0x80
4,13156,786700044185,-;[<ffff80000024bba0>] do_writev+0x60/0xf0
4,13157,786700044188,-;[<ffff80000024cf10>] SyS_writev+0x38/0x48
4,13158,786700044191,-;[<ffff8000000854e0>] __sys_trace_return+0x0/0x4
0,13159,786724246336,-;NMI watchdog: BUG: soft lockup - CPU#34 stuck for 23s! [sh:42014]
4,13160,786724253525,c;Modules linked in: xt_REDIRECT nf_nat_redirect xt_tcpudp xt_owner xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth ip_set_bitmap_port ip_set_hash_ipportnet ip_set_hash_ipport ip_set_hash_ipportip ip_set dummy ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle xt_comment xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xfrm_user xfrm_algo xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ip_tables x_tables br_netfilter bridge stp llc aufs rpcsec_gss_krb5 auth_rpcgss nfsv4 nfs lockd grace fscache overlay joydev igb ixgbe vxlan ip6_udp_tunnel udp_tunnel mdio shpchp dms5013a crc32_arm64 sunrpc autofs4
4,13161,786724253591,+; megaraid_sas
4,13162,786724253593,-;
4,13163,786724253597,-;CPU: 34 PID: 42014 Comm: sh Tainted: G             L  4.4.131-20190726.kylin.server-generic #kylin
4,13164,786724253599,-;Hardware name: GreatWall \xe6\x5cx93\x5cx8e\xe5\xa4\xa9DF720/N/A, BIOS 601FBE23 Dec  3 2019
4,13165,786724253602,-;task: ffff80835f0ae140 ti: ffff808350c48000 task.ti: ffff808350c48000
4,13166,786724253609,-;PC is at _raw_spin_lock_irqsave+0x1c/0x60
4,13167,786724253614,-;LR is at remove_wait_queue+0x24/0x60
4,13168,786724253616,-;pc : [<ffff800000a15034>] lr : [<ffff800000106ee4>] pstate: 20000145
4,13169,786724253618,-;sp : ffff808350c4bb90
4,13170,786724253619,-;x29: ffff808350c4bb90 x28: ffff84834e2e0800 
4,13171,786724253623,-;x27: ffff8000004db870 x26: ffff84834e2e08d8 
4,13172,786724253626,-;x25: ffff878347729300 x24: ffff800000a7b488 
4,13173,786724253628,-;x23: 0000000000000000 x22: 0000000000000000 
4,13174,786724253631,-;x21: ffff84834d289400 x20: ffff84834e2e0a30 
4,13175,786724253634,-;x19: ffff84834e2e0a30 x18: 2000000000000000 
4,13176,786724253636,-;x17: 0000ffff7921d540 x16: ffff80000024ced8 
4,13177,786724253639,-;x15: 000000000d39ad3d x14: 0000ffff79277f50 
4,13178,786724253642,-;x13: 0000000000000018 x12: 000000000d39ad3c 
4,13179,786724253644,-;x11: 00000000ffffffff x10: ffff808350c48000 
4,13180,786724253647,-;x9 : 000000007fff0000 x8 : 0001000000000000 
4,13181,786724253650,-;x7 : 0000000000000001 x6 : ffff84834d289400 
4,13182,786724253652,-;x5 : ffff84834d289400 x4 : 0001000000000000 
4,13183,786724253655,-;x3 : ffff84834e2e0a38 x2 : ffff84834d289400 
4,13184,786724253657,-;x1 : ffff808350c4bc48 x0 : 0000000000000140 
4,13185,786724253660,-;
0,13186,786752246332,-;NMI watchdog: BUG: soft lockup - CPU#34 stuck for 23s! [sh:42014]
4,13187,786752253522,c;Modules linked in: xt_REDIRECT nf_nat_redirect xt_tcpudp xt_owner xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth ip_set_bitmap_port ip_set_hash_ipportnet ip_set_hash_ipport ip_set_hash_ipportip ip_set dummy ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle xt_comment xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xfrm_user xfrm_algo xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ip_tables x_tables br_netfilter bridge stp llc aufs rpcsec_gss_krb5 auth_rpcgss nfsv4 nfs lockd grace fscache overlay joydev igb ixgbe vxlan ip6_udp_tunnel udp_tunnel mdio shpchp dms5013a crc32_arm64 sunrpc autofs4
4,13188,786752253589,+; megaraid_sas
4,13189,786752253590,-;
4,13190,786752253595,-;CPU: 34 PID: 42014 Comm: sh Tainted: G             L  4.4.131-20190726.kylin.server-generic #kylin
4,13191,786752253597,-;Hardware name: GreatWall \xe6\x5cx93\x5cx8e\xe5\xa4\xa9DF720/N/A, BIOS 601FBE23 Dec  3 2019
4,13192,786752253600,-;task: ffff80835f0ae140 ti: ffff808350c48000 task.ti: ffff808350c48000
4,13193,786752253608,-;PC is at ldsem_up_read+0x30/0x60
4,13194,786752253611,-;LR is at tty_ldisc_deref+0x24/0x30
4,13195,786752253613,-;pc : [<ffff8000004e22a0>] lr : [<ffff8000004df924>] pstate: 60000145
4,13196,786752253614,-;sp : ffff808350c4bc30
4,13197,786752253616,-;x29: ffff808350c4bc30 x28: ffff84834e2e0800 
4,13198,786752253620,-;x27: ffff8000004db870 x26: 0000000000000042 
4,13199,786752253623,-;x25: 0000000000000120 x24: 0000000000000000 
4,13200,786752253625,-;x23: ffff84834864b0c0 x22: ffff878347729300 
4,13201,786752253628,-;x21: 0000ffff792789f7 x20: 0000000000000000 
4,13202,786752253631,-;x19: ffff84834e2e0828 x18: 2000000000000000 
4,13203,786752253633,-;x17: 0000ffff7921d540 x16: ffff80000024ced8 
4,13204,786752253636,-;x15: 000000000d39ad3d x14: 0000ffff79277f50 
4,13205,786752253638,-;x13: 0000000000000018 x12: 000000000d39ad3c 
4,13206,786752253641,-;x11: 00000000ffffffff x10: ffff808350c48000 
4,13207,786752253643,-;x9 : 000000007fff0000 x8 : 0001000000000000 
4,13208,786752253646,-;x7 : 0000000000000001 x6 : ffff84834d289400 
4,13209,786752253649,-;x5 : dead000000000100 x4 : 0000000000000004 
4,13210,786752253651,-;x3 : 0000000000000000 x2 : 0000000000000000 
4,13211,786752253654,-;x1 : 0000000000000000 x0 : ffffffffffffffff 
4,13212,786752253656,-;
0,13213,786780246331,-;NMI watchdog: BUG: soft lockup - CPU#34 stuck for 23s! [sh:42014]
4,13214,786780253520,c;Modules linked in: xt_REDIRECT nf_nat_redirect xt_tcpudp xt_owner xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth ip_set_bitmap_port ip_set_hash_ipportnet ip_set_hash_ipport ip_set_hash_ipportip ip_set dummy ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle xt_comment xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xfrm_user xfrm_algo xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ip_tables x_tables br_netfilter bridge stp llc aufs rpcsec_gss_krb5 auth_rpcgss nfsv4 nfs lockd grace fscache overlay joydev igb ixgbe vxlan ip6_udp_tunnel udp_tunnel mdio shpchp dms5013a crc32_arm64 sunrpc autofs4
4,13215,786780253586,+; megaraid_sas
4,13216,786780253588,-;
4,13217,786780253591,-;CPU: 34 PID: 42014 Comm: sh Tainted: G             L  4.4.131-20190726.kylin.server-generic #kylin
4,13218,786780253594,-;Hardware name: GreatWall \xe6\x5cx93\x5cx8e\xe5\xa4\xa9DF720/N/A, BIOS 601FBE23 Dec  3 2019
4,13219,786780253597,-;task: ffff80835f0ae140 ti: ffff808350c48000 task.ti: ffff808350c48000
4,13220,786780253603,-;PC is at down_read+0x2c/0x58
4,13221,786780253606,-;LR is at n_tty_write+0x9c/0x460
4,13222,786780253609,-;pc : [<ffff800000a13784>] lr : [<ffff8000004db90c>] pstate: 80000145
4,13223,786780253610,-;sp : ffff808350c4bbb0
4,13224,786780253612,-;x29: ffff808350c4bbb0 x28: ffff84834e2e0800 
4,13225,786780253615,-;x27: ffff8000004db870 x26: ffff84834e2e08d8 
4,13226,786780253618,-;x25: ffff878347729300 x24: 0000000000000000 
4,13227,786780253621,-;x23: ffff84834864b0c0 x22: ffff878347729300 
4,13228,786780253624,-;x21: ffff84834e2e0a30 x20: 0000000000000000 
4,13229,786780253626,-;x19: ffff84834e2e08d8 x18: 2000000000000000 
4,13230,786780253629,-;x17: 0000ffff7921d540 x16: ffff80000024ced8 
4,13231,786780253632,-;x15: 000000000d39ad3d x14: 0000ffff79277f50 
4,13232,786780253634,-;x13: 0000000000000018 x12: 000000000d39ad3c 
4,13233,786780253637,-;x11: 00000000ffffffff x10: ffff808350c48000 
4,13234,786780253639,-;x9 : 000000007fff0000 x8 : 0001000000000000 
4,13235,786780253642,-;x7 : 0000000000000001 x6 : ffff84834d289400 
4,13236,786780253645,-;x5 : ffff84834d289400 x4 : 0001000000000000 
4,13237,786780253647,-;x3 : ffff80835f0ae140 x2 : 0000000000008a30 
4,13238,786780253650,-;x1 : 0000000000000000 x0 : 0000000000000001 
4,13239,786780253652,-;
6,13240,786804718407,-;docker0: port 1(veth71f29ad) entered disabled state
6,13241,786804718628,-;vethb794498: renamed from eth0
6,13242,786804920203,-;docker0: port 1(veth71f29ad) entered disabled state
6,13243,786804941153,-;device veth71f29ad left promiscuous mode
6,13244,786804941159,-;docker0: port 1(veth71f29ad) entered disabled state

root@node-4:~# top 

top - 14:15:37 up 9 days,  2:23,  4 users,  load average: 3.50, 2.04, 1.99
Tasks: 724 total,   6 running, 718 sleeping,   0 stopped,   0 zombie
%Cpu(s):  0.9 us,  0.5 sy,  0.0 ni, 98.5 id,  0.1 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem : 26587347+total, 21568915+free, 10836992 used, 39347328 buff/cache
KiB Swap:        0 total,        0 free,        0 used. 21257120+avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND                                                                                                                                 
50269 root      20   0    2048     64      0 R 100.0  0.0   0:14.95 sh

cpu soft lockup

ningshuofu commented 2 years ago

问题解决了吗，我也遇到了类似问题

ningshuofu commented 2 years ago

Oct 30 13:00:16 stor01 kernel: [9318563.670802][11] connection3:0: ping timeout of 5 secs expired, recv timeout 5, last rx 6624530701, last ping 6624531952, now 6624533204 Oct 30 13:00:16 stor01 kernel: [9318563.670810][11] connection4:0: ping timeout of 5 secs expired, recv timeout 5, last rx 6624530700, last ping 6624531952, now 6624533204 Oct 30 13:00:16 stor01 kernel: [9318563.670837][11] connection4:0: detected conn error (1022) Oct 30 13:00:16 stor01 kernel: [9318563.671004][11] connection3:0: detected conn error (1022) Oct 30 13:00:16 stor01 kernel: [9318563.921397][11] sd 12:0:0:0: [sdk] tag#3 FAILED Result: hostbyte=DID_TRANSPORT_DISRUPTED driverbyte=DRIVER_OK Oct 30 13:00:16 stor01 kernel: [9318563.921411][11] sd 12:0:0:0: [sdk] tag#3 CDB: Test Unit Ready 00 00 00 00 00 00 Oct 30 13:00:41 stor01 kernel: [9318588.678858][15] lockd: cannot monitor node-11 Oct 30 13:01:09 stor01 kernel: [9318617.066792][10] INFO: rcu_sched detected stalls on CPUs/tasks: Oct 30 13:01:09 stor01 kernel: [9318617.066863][10] 7-...: (1 GPs behind) idle=c69/140000000000000/0 softirq=430020531/430020531 fqs=14937 Oct 30 13:01:09 stor01 kernel: [9318617.066924][10] (detected by 9, t=15002 jiffies, g=432840556, c=432840555, q=33347) Oct 30 13:01:09 stor01 kernel: [9318617.066982][10] Task dump for CPU 7: Oct 30 13:01:09 stor01 kernel: [9318617.066986][10] celery R running task 0 4044789 3200 0x0000000a Oct 30 13:01:09 stor01 kernel: [9318617.066995][10] Call trace: Oct 30 13:01:09 stor01 kernel: [9318617.067027][10] [] switch_to+0xa0/0xb8 Oct 30 13:01:09 stor01 kernel: [9318617.067032][10] [] 0xffff8010d1e93800 Oct 30 13:01:41 stor01 kernel: [9318648.838846][15] lockd: cannot monitor node-11 Oct 30 13:02:41 stor01 kernel: [9318708.998854][12] lockd: cannot monitor node-11 Oct 30 13:02:53 stor01 kernel: [9318720.835612][ 5] INFO: task 9_dirty_io_sche:3087578 blocked for more than 120 seconds. Oct 30 13:02:53 stor01 kernel: [9318720.835685][ 5] Tainted: G OE 4.4.131-20201204.kylin.server-generic #kylin Oct 30 13:02:53 stor01 kernel: [9318720.835740][ 5] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Oct 30 13:02:53 stor01 kernel: [9318720.835793][ 5] 9_dirty_io_sche D ffff800000086408 0 3087578 2971 0x00000000 Oct 30 13:02:53 stor01 kernel: [9318720.835802][ 5] Call trace: Oct 30 13:02:53 stor01 kernel: [9318720.835833][ 5] [] switch_to+0xa0/0xb8 Oct 30 13:02:53 stor01 kernel: [9318720.835842][ 5] [] __schedule+0x24c/0x730 Oct 30 13:02:53 stor01 kernel: [9318720.835847][ 5] [] schedule+0x30/0x88 Oct 30 13:02:53 stor01 kernel: [9318720.835853][ 5] [] schedule_timeout+0x1f0/0x250 Oct 30 13:02:53 stor01 kernel: [9318720.835858][ 5] [] wait_for_common+0xbc/0x170 Oct 30 13:02:53 stor01 kernel: [9318720.835863][ 5] [] wait_for_completion+0x28/0x38 Oct 30 13:02:53 stor01 kernel: [9318720.835870][ 5] [] wait_rcu_gp+0x104/0x138 Oct 30 13:02:53 stor01 kernel: [9318720.835875][ 5] [] synchronize_sched+0x54/0x68 Oct 30 13:02:53 stor01 kernel: [9318720.835881][ 5] [] expand_files+0x154/0x230 Oct 30 13:02:53 stor01 kernel: [9318720.835885][ 5] [] alloc_fd+0x50/0x1e8 Oct 30 13:02:53 stor01 kernel: [9318720.835889][ 5] [] get_unused_fd_flags+0x38/0x48 Oct 30 13:02:53 stor01 kernel: [9318720.835895][ 5] [] do_sys_open+0x118/0x2a8 Oct 30 13:02:53 stor01 kernel: [9318720.835900][ 5] [] SyS_openat+0x3c/0x50 Oct 30 13:02:53 stor01 kernel: [9318720.835904][ 5] [] el0_svc_naked+0x24/0x28