search

openwrt / mt76

mac80211 driver for MediaTek MT76x0e, MT76x2e, MT7603, MT7615, MT7628 and MT7688
750 stars 342 forks source link

kernel crash on wireless module reload #599

Open bol-van opened 3 years ago

bol-van commented 3 years ago

I'm using TP-link archer c6u (MT7621) Recently I've upgraded to snapshot openwrt version with 5.10 kernel and latest mt76 snapshot now this script causes in most cases kernel panics

#!/bin/sh

wifi down
/etc/init.d/wpad stop
# have to wait 1 sec or 5.10 snapshot kernel crashes
rmmod mt7603e
rmmod mt7615e
rmmod mt7615_common
rmmod mt76_connac_lib
rmmod mt76
rmmod mac80211
rmmod cfg80211
rmmod compat

kmodloader
/etc/init.d/wpad start
wifi

[  129.383166] kmodloader: loading kernel modules from /etc/modules.d/*
[  129.411502] Loading modules backported from Linux version v5.10.68-0-g4d8524048a35
[  129.419154] Backport generated by backports.git v5.10.68-1-0-ga4f9ba32
[  129.471431] Unhandled kernel unaligned access[#1]:
[  129.476355] CPU: 2 PID: 21 Comm: ksoftirqd/2 Not tainted 5.10.64 #0
[  129.480521] mac80211: Unknown symbol path_refresh_time_ops (err -2)
[  129.482655] $ 0   : 00000000 00000001 81024b80 00000040
[  129.494264] $ 4   : 8101e2a4 00000006 ffffbd5e 00000000
[  129.499547] $ 8   : 8506a580 00000040 0000000c 00000000
[  129.504797] $12   : 80e36310 00000000 ffffffff 006e6177
[  129.510035] $16   : 000e0001 000e0001 8101e2a4 80944b80
[  129.515278] $20   : 8087c47c 00000040 3fffffff 8086e040
[  129.520517] $24   : 840f3c40 800ec264                  
[  129.525750] $28   : 80c98000 80c99e20 8093e310 806f135c
[  129.530999] Hi    : 00000000
[  129.533883] Lo    : 00000014
[  129.536778] epc   : 80035cf4 tasklet_action_common.constprop.0+0xb8/0x17c
[  129.543571] ra    : 806f135c __do_softirq+0x164/0x368
[  129.548602] Status: 1100fc03 KERNEL EXL IE 
[  129.552794] Cause : 40800010 (ExcCode 04)
[  129.556784] BadVA : 000e0001
[  129.559662] PrId  : 0001992f (MIPS 1004Kc)
[  129.563747] Modules linked in: cfg80211 compat ipobfs pppoe ppp_async wireguard pppox ppp_generic libchacha20poly1305 libblake2s ipt_REJECT xt_u32 xt_time xt_tcpudp xt_tcpmss xt_string xt_statistic xt_state xt_recent xt_quota xt_pkttype xt_owner xt_nat xt_multiport xt_mark xt_mac xt_limit xt_length xt_ipv4options xt_iprange xt_iface xt_hl xt_helper xt_ecn xt_dscp xt_conntrack xt_connmark xt_connlimit xt_connbytes xt_comment xt_cgroup xt_bpf xt_addrtype xt_TCPMSS xt_REDIRECT xt_NFQUEUE xt_NFLOG xt_NETMAP xt_MASQUERADE xt_LOG xt_IPMARK xt_HL xt_FLOWOFFLOAD xt_DSCP xt_CT xt_CLASSIFY ts_kmp ts_fsm ts_bm slhc poly1305_mips nfnetlink_queue nfnetlink_log nf_reject_ipv4 nf_log_ipv4 nf_flow_table nf_conntrack_netlink nf_conncount libcurve25519_generic libblake2s_generic iptable_raw iptable_nat iptable_mangle iptable_filter ipt_ECN ip_tables hwmon crc_ccitt compat_xtables chacha_mips sch_tbf sch_ingress sch_htb sch_hfsc em_u32 cls_u32 cls_tcindex cls_route cls_matchall cls_fw cls_flow cls_basic
[  129.564376]  act_skbedit act_mirred act_gact ledtrig_usbport xt_set ip_set_list_set ip_set_hash_netportnet ip_set_hash_netport ip_set_hash_netnet ip_set_hash_netiface ip_set_hash_net ip_set_hash_mac ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_hash_ipmark ip_set_hash_ip ip_set_bitmap_port ip_set_bitmap_ipmac ip_set_bitmap_ip ip_set nfnetlink ip6table_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip6t_NPT ip6t_rt ip6t_mh ip6t_ipv6header ip6t_hbh ip6t_frag ip6t_eui64 ip6t_ah nf_log_ipv6 nf_log_common ip6table_mangle ip6table_filter ip6_tables ip6t_REJECT x_tables nf_reject_ipv6 nfsv4 nfsd nfs nfs_ssc ip6_udp_tunnel udp_tunnel sit netlink_diag tunnel4 ip_tunnel rpcsec_gss_krb5 auth_rpcgss oid_registry lockd sunrpc grace cifs dns_resolver dm_mirror dm_region_hash dm_log dm_crypt dm_mod dax nls_utf8 zram zsmalloc sha512_generic sha1_generic seqiv jitterentropy_rng drbg md5 md4 kpp hmac ecb des_generic libdes cts cmac arc4 cbc sha256_generic libsha256
[  129.651772]  encrypted_keys trusted tpm usb_storage leds_gpio xhci_plat_hcd xhci_pci xhci_mtk xhci_hcd sd_mod scsi_mod gpio_button_hotplug ext4 mbcache jbd2 usbcore nls_base usb_common crc32c_generic [last unloaded: compat]
[  129.757760] Process ksoftirqd/2 (pid: 21, threadinfo=db228510, task=53016988, tls=00000000)
[  129.766080] Stack : 80c99e20 80c99e20 80c99e28 80c99e28 8086e04c 8086e058 00000040 00000006
[  129.774456]         00000007 00000100 8087c47c 806f135c ffffffff 006e6177 80c90c00 00000001
[  129.782828]         80881dc4 00000002 8093e2ac 80944b80 00000005 ffffbd5f 80880000 8078fdac
[  129.791201]         04208040 8078fea8 80c67e00 00000001 80881dc4 00000002 80c67e00 80790000
[  129.799573]         80a50000 00000001 00000000 80035f78 1100fc03 00000000 00000014 806ece90
[  129.807945]         ...
[  129.810392] Call Trace:
[  129.812848] [<80035cf4>] tasklet_action_common.constprop.0+0xb8/0x17c
[  129.819295] [<806f135c>] __do_softirq+0x164/0x368
[  129.823987] [<80035f78>] run_ksoftirqd+0x5c/0x70
[  129.828602] [<800584c4>] smpboot_thread_fn+0x100/0x1d4
[  129.833734] [<80052214>] kthread+0x13c/0x144
[  129.838001] [<800030f8>] ret_from_kernel_thread+0x14/0x1c
[  129.843384] 
[  129.844867] Code: 12200026  8fbf002c  02208025 <8e310000> c2020004  34430002  e2030004  1060fffc  00000000 
[  129.854635] 
[  129.856544] ---[ end trace 4b413040acfb0dfd ]---
[  129.861281] Kernel panic - not syncing: Fatal exception in interrupt
[  129.867650] ------------[ cut here ]------------
[  129.872313] WARNING: CPU: 2 PID: 21 at kernel/smp.c:633 smp_call_function_many_cond+0x438/0x454
[  129.880971] Modules linked in: cfg80211 compat ipobfs pppoe ppp_async wireguard pppox ppp_generic libchacha20poly1305 libblake2s ipt_REJECT xt_u32 xt_time xt_tcpudp xt_tcpmss xt_string xt_statistic xt_state xt_recent xt_quota xt_pkttype xt_owner xt_nat xt_multiport xt_mark xt_mac xt_limit xt_length xt_ipv4options xt_iprange xt_iface xt_hl xt_helper xt_ecn xt_dscp xt_conntrack xt_connmark xt_connlimit xt_connbytes xt_comment xt_cgroup xt_bpf xt_addrtype xt_TCPMSS xt_REDIRECT xt_NFQUEUE xt_NFLOG xt_NETMAP xt_MASQUERADE xt_LOG xt_IPMARK xt_HL xt_FLOWOFFLOAD xt_DSCP xt_CT xt_CLASSIFY ts_kmp ts_fsm ts_bm slhc poly1305_mips nfnetlink_queue nfnetlink_log nf_reject_ipv4 nf_log_ipv4 nf_flow_table nf_conntrack_netlink nf_conncount libcurve25519_generic libblake2s_generic iptable_raw iptable_nat iptable_mangle iptable_filter ipt_ECN ip_tables hwmon crc_ccitt compat_xtables chacha_mips sch_tbf sch_ingress sch_htb sch_hfsc em_u32 cls_u32 cls_tcindex cls_route cls_matchall cls_fw cls_flow cls_basic
[  129.881601]  act_skbedit act_mirred act_gact ledtrig_usbport xt_set ip_set_list_set ip_set_hash_netportnet ip_set_hash_netport ip_set_hash_netnet ip_set_hash_netiface ip_set_hash_net ip_set_hash_mac ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_hash_ipmark ip_set_hash_ip ip_set_bitmap_port ip_set_bitmap_ipmac ip_set_bitmap_ip ip_set nfnetlink ip6table_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip6t_NPT ip6t_rt ip6t_mh ip6t_ipv6header ip6t_hbh ip6t_frag ip6t_eui64 ip6t_ah nf_log_ipv6 nf_log_common ip6table_mangle ip6table_filter ip6_tables ip6t_REJECT x_tables nf_reject_ipv6 nfsv4 nfsd nfs nfs_ssc ip6_udp_tunnel udp_tunnel sit netlink_diag tunnel4 ip_tunnel rpcsec_gss_krb5 auth_rpcgss oid_registry lockd sunrpc grace cifs dns_resolver dm_mirror dm_region_hash dm_log dm_crypt dm_mod dax nls_utf8 zram zsmalloc sha512_generic sha1_generic seqiv jitterentropy_rng drbg md5 md4 kpp hmac ecb des_generic libdes cts cmac arc4 cbc sha256_generic libsha256
[  129.969000]  encrypted_keys trusted tpm usb_storage leds_gpio xhci_plat_hcd xhci_pci xhci_mtk xhci_hcd sd_mod scsi_mod gpio_button_hotplug ext4 mbcache jbd2 usbcore nls_base usb_common crc32c_generic [last unloaded: compat]
[  130.074982] CPU: 2 PID: 21 Comm: ksoftirqd/2 Tainted: G      D           5.10.64 #0
[  130.082608] Stack : 80a70000 00000000 8093e310 80084ec4 808f0000 807d7d94 00000000 00000000
[  130.090986]         80c99b74 80a50000 8079effc 80c90fc8 8087eec7 00000001 80c99b18 7c98f3bb
[  130.099360]         00000000 00000000 8079effc 80c999b8 ffffefff 00000000 ffffffea 00000000
[  130.107732]         80c999c4 00000217 80884a68 ffffffff 00000000 00000000 00000000 807a0000
[  130.116103]         00000009 806f135c 3fffffff 00000000 00000018 804220e4 00000008 80a50008
[  130.124472]         ...
[  130.126920] Call Trace:
[  130.129391] [<800081e0>] show_stack+0x30/0x100
[  130.133844] [<8039a29c>] dump_stack+0x9c/0xcc
[  130.138215] [<80031640>] __warn+0xc0/0x12c
[  130.142305] [<80031708>] warn_slowpath_fmt+0x5c/0xac
[  130.147281] [<800c3938>] smp_call_function_many_cond+0x438/0x454
[  130.153274] [<800c3998>] smp_call_function+0x24/0x30
[  130.158230] [<800310b4>] panic+0x120/0x320
[  130.162319] [<800084c8>] die+0x134/0x13c
[  130.166236] [<8000a8c8>] do_ade+0x998/0xac8
[  130.170413] [<80003a94>] handle_adel_int+0x30/0x3c
[  130.175214] [<806f135c>] __do_softirq+0x164/0x368
[  130.179904] 
[  130.181393] ---[ end trace 4b413040acfb0dfe ]---
[  130.186021] Rebooting in 3 seconds..

[   88.241516] CPU 2 Unable to handle kernel paging request at virtual address 00000994, epc == 80035cf4, ra == 806f135c
[   88.252230] Oops[#1]:
[   88.254530] CPU: 2 PID: 21 Comm: ksoftirqd/2 Not tainted 5.10.64 #0
[   88.260804] $ 0   : 00000000 80a50008 81024b80 00000040
[   88.266074] $ 4   : 8101e2a4 00000006 ffffad42 ffff00fe
[   88.271313] $ 8   : 80c99fe0 0000fc00 00000001 00000014
[   88.276542] $12   : 81024f60 00000400 00000000 00000000
[   88.281773] $16   : 00000994 00000994 8101e2a4 80944b80
[   88.287023] $20   : 8087c47c 00000040 3fffffff 8086e040
[   88.292302] $24   : 00000001 025016a9                  
[   88.297559] $28   : 80c98000 80c99e20 8093e310 806f135c
[   88.302792] Hi    : 0002acaa
[   88.305660] Lo    : 3104b755
[   88.308576] epc   : 80035cf4 tasklet_action_common.constprop.0+0xb8/0x17c
[   88.315404] ra    : 806f135c __do_softirq+0x164/0x368
[   88.320474] Status: 1100fc03 KERNEL EXL IE 
[   88.323768] mt7621-pci 1e140000.pcie: bus=1 slot=0 irq=24
[   88.324701] Cause : 40800008 (ExcCode 02)
[   88.330375] mt7603e 0000:01:00.0: ASIC revision: 76030010
[   88.334054] BadVA : 00000994
[   88.342286] PrId  : 0001992f (MIPS 1004Kc)
[   88.346378] Modules linked in: mt7603e(+) mt76_connac_lib mt76 mac80211 cfg80211 compat ipobfs pppoe ppp_async wireguard pppox ppp_generic libchacha20poly1305 libblake2s ipt_REJECT xt_u32 xt_time xt_tcpudp xt_tcpmss xt_string xt_statistic xt_state xt_recent xt_quota xt_pkttype xt_owner xt_nat xt_multiport xt_mark xt_mac xt_limit xt_length xt_ipv4options xt_iprange xt_iface xt_hl xt_helper xt_ecn xt_dscp xt_conntrack xt_connmark xt_connlimit xt_connbytes xt_comment xt_cgroup xt_bpf xt_addrtype xt_TCPMSS xt_REDIRECT xt_NFQUEUE xt_NFLOG xt_NETMAP xt_MASQUERADE xt_LOG xt_IPMARK xt_HL xt_FLOWOFFLOAD xt_DSCP xt_CT xt_CLASSIFY ts_kmp ts_fsm ts_bm slhc poly1305_mips nfnetlink_queue nfnetlink_log nf_reject_ipv4 nf_log_ipv4 nf_flow_table nf_conntrack_netlink nf_conncount libcurve25519_generic libblake2s_generic iptable_raw iptable_nat iptable_mangle iptable_filter ipt_ECN ip_tables hwmon crc_ccitt compat_xtables chacha_mips sch_tbf sch_ingress sch_htb sch_hfsc em_u32 cls_u32 cls_tcindex cls_route
[   88.347017]  cls_matchall cls_fw cls_flow cls_basic act_skbedit act_mirred act_gact ledtrig_usbport xt_set ip_set_list_set ip_set_hash_netportnet ip_set_hash_netport ip_set_hash_netnet ip_set_hash_netiface ip_set_hash_net ip_set_hash_mac ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_hash_ipmark ip_set_hash_ip ip_set_bitmap_port ip_set_bitmap_ipmac ip_set_bitmap_ip ip_set nfnetlink ip6table_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip6t_NPT ip6t_rt ip6t_mh ip6t_ipv6header ip6t_hbh ip6t_frag ip6t_eui64 ip6t_ah nf_log_ipv6 nf_log_common ip6table_mangle ip6table_filter ip6_tables ip6t_REJECT x_tables nf_reject_ipv6 nfsv4 nfsd nfs nfs_ssc ip6_udp_tunnel udp_tunnel sit netlink_diag tunnel4 ip_tunnel rpcsec_gss_krb5 auth_rpcgss oid_registry lockd sunrpc grace cifs dns_resolver dm_mirror dm_region_hash dm_log dm_crypt dm_mod dax nls_utf8 zram zsmalloc sha512_generic sha1_generic seqiv jitterentropy_rng drbg md5 md4 kpp hmac ecb des_generic libdes cts cmac arc4
[   88.434591]  cbc sha256_generic libsha256 encrypted_keys trusted tpm usb_storage leds_gpio xhci_plat_hcd xhci_pci xhci_mtk xhci_hcd sd_mod scsi_mod gpio_button_hotplug ext4 mbcache jbd2 usbcore nls_base usb_common crc32c_generic [last unloaded: compat]
[   88.543958] Process ksoftirqd/2 (pid: 21, threadinfo=7c376de0, task=694c467a, tls=00000000)
[   88.552274] Stack : 80c90c00 00000000 80c98000 80c99e50 8093e310 8086e058 00000040 00000006
[   88.560646]         00000007 00000100 8087c47c 806f135c 80c99fe0 0000fc00 00000001 00000014
[   88.569019]         81024f60 00000400 8093e2ac 80944b80 00000004 ffffad43 80880000 8078fdac
[   88.577392]         04208040 8078fea8 80c67e00 00000001 80881dc4 00000002 80c67e00 80790000
[   88.585764]         80a50000 00000001 00000000 80035f78 1100fc03 0002acaa 3104b755 806ece90
[   88.594136]         ...
[   88.596583] Call Trace:
[   88.599041] [<80035cf4>] tasklet_action_common.constprop.0+0xb8/0x17c
[   88.605490] [<806f135c>] __do_softirq+0x164/0x368
[   88.610182] [<80035f78>] run_ksoftirqd+0x5c/0x70
[   88.614795] [<800584c4>] smpboot_thread_fn+0x100/0x1d4
[   88.619928] [<80052214>] kthread+0x13c/0x144
[   88.624192] [<800030f8>] ret_from_kernel_thread+0x14/0x1c
[   88.629576] 
[   88.631061] Code: 12200026  8fbf002c  02208025 <8e310000> c2020004  34430002  e2030004  1060fffc  00000000 
[   88.640828] 
[   88.642661] ---[ end trace 2843b36dae0f2c0f ]---
[   88.644609] mt7603e 0000:01:00.0: Firmware Version: ap_pcie
[   88.647380] Kernel panic - not syncing: Fatal exception in interrupt
[   88.652901] mt7603e 0000:01:00.0: Build Time: 20160107100755
[   88.659199] ------------[ cut here ]------------
[   88.659254] WARNING: CPU: 2 PID: 21 at kernel/smp.c:633 smp_call_function_many_cond+0x438/0x454
[   88.659267] Modules linked in: mt7603e(+) mt76_connac_lib mt76 mac80211 cfg80211 compat ipobfs pppoe ppp_async wireguard pppox ppp_generic libchacha20poly1305 libblake2s ipt_REJECT xt_u32 xt_time xt_tcpudp xt_tcpmss xt_string xt_statistic xt_state xt_recent xt_quota xt_pkttype xt_owner xt_nat xt_multiport xt_mark xt_mac xt_limit xt_length xt_ipv4options xt_iprange xt_iface xt_hl xt_helper xt_ecn xt_dscp xt_conntrack xt_connmark xt_connlimit xt_connbytes xt_comment xt_cgroup xt_bpf xt_addrtype xt_TCPMSS xt_REDIRECT xt_NFQUEUE xt_NFLOG xt_NETMAP xt_MASQUERADE xt_LOG xt_IPMARK xt_HL xt_FLOWOFFLOAD xt_DSCP xt_CT xt_CLASSIFY ts_kmp ts_fsm ts_bm slhc poly1305_mips nfnetlink_queue nfnetlink_log nf_reject_ipv4 nf_log_ipv4 nf_flow_table nf_conntrack_netlink nf_conncount libcurve25519_generic libblake2s_generic iptable_raw iptable_nat iptable_mangle iptable_filter ipt_ECN ip_tables hwmon crc_ccitt compat_xtables chacha_mips sch_tbf sch_ingress sch_htb sch_hfsc em_u32 cls_u32 cls_tcindex cls_route
[   88.678796]  cls_matchall cls_fw cls_flow cls_basic act_skbedit act_mirred act_gact ledtrig_usbport xt_set ip_set_list_set ip_set_hash_netportnet ip_set_hash_netport ip_set_hash_netnet ip_set_hash_netiface ip_set_hash_net ip_set_hash_mac ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_hash_ipmark ip_set_hash_ip ip_set_bitmap_port ip_set_bitmap_ipmac ip_set_bitmap_ip ip_set nfnetlink ip6table_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip6t_NPT ip6t_rt ip6t_mh ip6t_ipv6header ip6t_hbh ip6t_frag ip6t_eui64 ip6t_ah nf_log_ipv6 nf_log_common ip6table_mangle ip6table_filter ip6_tables ip6t_REJECT x_tables nf_reject_ipv6 nfsv4 nfsd nfs nfs_ssc ip6_udp_tunnel udp_tunnel sit netlink_diag tunnel4 ip_tunnel rpcsec_gss_krb5 auth_rpcgss oid_registry lockd sunrpc grace cifs dns_resolver dm_mirror dm_region_hash dm_log dm_crypt dm_mod dax nls_utf8 zram zsmalloc sha512_generic sha1_generic seqiv jitterentropy_rng drbg md5 md4 kpp hmac ecb des_generic libdes cts cmac arc4
[   88.766359]  cbc sha256_generic libsha256 encrypted_keys trusted tpm usb_storage leds_gpio xhci_plat_hcd xhci_pci xhci_mtk xhci_hcd sd_mod scsi_mod gpio_button_hotplug ext4 mbcache jbd2 usbcore nls_base usb_common crc32c_generic [last unloaded: compat]
[   88.875725] CPU: 2 PID: 21 Comm: ksoftirqd/2 Tainted: G      D           5.10.64 #0
[   88.883349] Stack : 80a70000 00000000 8093e310 80084ec4 808f0000 807d7d94 00000000 00000000
[   88.891721]         80c99b6c 80a50000 8079effc 80c90fc8 8087eec7 00000001 80c99b10 00bd5924
[   88.900094]         00000000 00000000 8079effc 80c999b0 ffffefff 00000000 ffffffea 00000000
[   88.908465]         80c999bc 000001e7 80884a68 ffffffff 00000000 00000000 00000000 807a0000
[   88.916836]         00000009 80c90c00 00000000 00000000 00000018 804220e4 01800117 01800157
[   88.925208]         ...
[   88.927656] Call Trace:
[   88.930122] [<800081e0>] show_stack+0x30/0x100
[   88.934572] [<8039a29c>] dump_stack+0x9c/0xcc
[   88.938943] [<80031640>] __warn+0xc0/0x12c
[   88.943033] [<80031708>] warn_slowpath_fmt+0x5c/0xac
[   88.948016] [<800c3938>] smp_call_function_many_cond+0x438/0x454
[   88.954006] [<800c3998>] smp_call_function+0x24/0x30
[   88.958962] [<800310b4>] panic+0x120/0x320
[   88.963051] [<800084c8>] die+0x134/0x13c
[   88.966988] [<8001c5d0>] do_page_fault+0x4e0/0x4f0
[   88.971766] [<80022658>] tlb_do_page_fault_0+0x118/0x120
[   88.977086] [<806f135c>] __do_softirq+0x164/0x368
[   88.981774] 
[   88.983259] ---[ end trace 2843b36dae0f2c10 ]---
[   88.987884] Rebooting in 3 seconds..

[   93.643107] Kernel bug detected[#1]:
[   93.646734] CPU: 2 PID: 21 Comm: ksoftirqd/2 Not tainted 5.10.64 #0
[   93.652989] $ 0   : 00000000 00000001 00000000 00000001
[   93.658231] $ 4   : 8101e2a4 00000006 ffffaf5f 00000000
[   93.663465] $ 8   : a269e4a2 00000008 8087c47c ffffffff
[   93.668697] $12   : 00000001 00000400 00000000 00000000
[   93.673929] $16   : 83da38fc 00000000 8101e2a4 80944b80
[   93.679161] $20   : 8087c47c 00000040 3fffffff 8086e040
[   93.684393] $24   : 815f30a0 00000000                  
[   93.689625] $28   : 80c98000 80c99e20 8093e310 806f135c
[   93.694857] Hi    : 0006a698
[   93.697728] Lo    : 5080d5e0
[   93.700618] epc   : 80035db4 tasklet_action_common.constprop.0+0x178/0x17c
[   93.707490] ra    : 806f135c __do_softirq+0x164/0x368
[   93.712520] Status: 1100fc03 KERNEL EXL IE 
[   93.716710] Cause : 50800024 (ExcCode 09)
[   93.720703] PrId  : 0001992f (MIPS 1004Kc)
[   93.724781] Modules linked in: ipobfs pppoe ppp_async wireguard pppox ppp_generic mt7615e mt7615_common mt76_connac_lib mt76 mac80211 libchacha20poly1305 libblake2s ipt_REJECT cfg80211 xt_u32 xt_time xt_tcpudp xt_tcpmss xt_string xt_statistic xt_state xt_recent xt_quota xt_pkttype xt_owner xt_nat xt_multiport xt_mark xt_mac xt_limit xt_length xt_ipv4options xt_iprange xt_iface xt_hl xt_helper xt_ecn xt_dscp xt_conntrack xt_connmark xt_connlimit xt_connbytes xt_comment xt_cgroup xt_bpf xt_addrtype xt_TCPMSS xt_REDIRECT xt_NFQUEUE xt_NFLOG xt_NETMAP xt_MASQUERADE xt_LOG xt_IPMARK xt_HL xt_FLOWOFFLOAD xt_DSCP xt_CT xt_CLASSIFY ts_kmp ts_fsm ts_bm slhc poly1305_mips nfnetlink_queue nfnetlink_log nf_reject_ipv4 nf_log_ipv4 nf_flow_table nf_conntrack_netlink nf_conncount libcurve25519_generic libblake2s_generic iptable_raw iptable_nat iptable_mangle iptable_filter ipt_ECN ip_tables hwmon crc_ccitt compat_xtables compat chacha_mips sch_tbf sch_ingress sch_htb sch_hfsc em_u32 cls_u32 cls_tcindex
[   93.725404]  cls_route cls_matchall cls_fw cls_flow cls_basic act_skbedit act_mirred act_gact ledtrig_usbport xt_set ip_set_list_set ip_set_hash_netportnet ip_set_hash_netport ip_set_hash_netnet ip_set_hash_netiface ip_set_hash_net ip_set_hash_mac ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_hash_ipmark ip_set_hash_ip ip_set_bitmap_port ip_set_bitmap_ipmac ip_set_bitmap_ip ip_set nfnetlink ip6table_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip6t_NPT ip6t_rt ip6t_mh ip6t_ipv6header ip6t_hbh ip6t_frag ip6t_eui64 ip6t_ah nf_log_ipv6 nf_log_common ip6table_mangle ip6table_filter ip6_tables ip6t_REJECT x_tables nf_reject_ipv6 nfsv4 nfsd nfs nfs_ssc ip6_udp_tunnel udp_tunnel sit netlink_diag tunnel4 ip_tunnel rpcsec_gss_krb5 auth_rpcgss oid_registry lockd sunrpc grace cifs dns_resolver dm_mirror dm_region_hash dm_log dm_crypt dm_mod dax nls_utf8 zram zsmalloc sha512_generic sha1_generic seqiv jitterentropy_rng drbg md5 md4 kpp hmac ecb des_generic libdes cts
[   93.813043]  cmac arc4 cbc sha256_generic libsha256 encrypted_keys trusted tpm usb_storage leds_gpio xhci_plat_hcd xhci_pci xhci_mtk xhci_hcd sd_mod scsi_mod gpio_button_hotplug ext4 mbcache jbd2 usbcore nls_base usb_common crc32c_generic [last unloaded: mt7603e]
[   93.923369] Process ksoftirqd/2 (pid: 21, threadinfo=5030145d, task=fbfce996, tls=00000000)
[   93.931684] Stack : 80c90c60 81024f60 80c98000 80c99e50 8093e310 8086e058 00000040 00000006
[   93.940057]         00000007 00000100 8087c47c 806f135c 8087c47c 80c90f64 80a50000 806ec7b8
[   93.948431]         81024f60 00000400 8093e2ac 80944b80 00000009 ffffaf60 80880000 8078fdac
[   93.956802]         04208040 8078fea8 80c67e00 00000001 80881dc4 00000002 80c67e00 80790000
[   93.965172]         80a50000 00000001 00000000 80035f78 1100fc03 0006a072 ff5f8b5f 806ece90
[   93.973543]         ...
[   93.975991] Call Trace:
[   93.978439] [<80035db4>] tasklet_action_common.constprop.0+0x178/0x17c
[   93.984955] [<806f135c>] __do_softirq+0x164/0x368
[   93.989649] [<80035f78>] run_ksoftirqd+0x5c/0x70
[   93.994263] [<800584c4>] smpboot_thread_fn+0x100/0x1d4
[   93.999396] [<80052214>] kthread+0x13c/0x144
[   94.003660] [<800030f8>] ret_from_kernel_thread+0x14/0x1c
[   94.009044] 
[   94.010529] Code: 8e040014  1000ffc9  00000000 <000c000d> 8f83000c  3c028088  2442c47c  00031880  00621821 
[   94.020296] 
[   94.022105] ---[ end trace 9d515235432dc444 ]---
[   94.026753] Kernel panic - not syncing: Fatal exception in interrupt
[   94.033130] ------------[ cut here ]------------
[   94.037796] WARNING: CPU: 2 PID: 21 at kernel/smp.c:633 smp_call_function_many_cond+0x438/0x454
[   94.046455] Modules linked in: ipobfs pppoe ppp_async wireguard pppox ppp_generic mt7615e mt7615_common mt76_connac_lib mt76 mac80211 libchacha20poly1305 libblake2s ipt_REJECT cfg80211 xt_u32 xt_time xt_tcpudp xt_tcpmss xt_string xt_statistic xt_state xt_recent xt_quota xt_pkttype xt_owner xt_nat xt_multiport xt_mark xt_mac xt_limit xt_length xt_ipv4options xt_iprange xt_iface xt_hl xt_helper xt_ecn xt_dscp xt_conntrack xt_connmark xt_connlimit xt_connbytes xt_comment xt_cgroup xt_bpf xt_addrtype xt_TCPMSS xt_REDIRECT xt_NFQUEUE xt_NFLOG xt_NETMAP xt_MASQUERADE xt_LOG xt_IPMARK xt_HL xt_FLOWOFFLOAD xt_DSCP xt_CT xt_CLASSIFY ts_kmp ts_fsm ts_bm slhc poly1305_mips nfnetlink_queue nfnetlink_log nf_reject_ipv4 nf_log_ipv4 nf_flow_table nf_conntrack_netlink nf_conncount libcurve25519_generic libblake2s_generic iptable_raw iptable_nat iptable_mangle iptable_filter ipt_ECN ip_tables hwmon crc_ccitt compat_xtables compat chacha_mips sch_tbf sch_ingress sch_htb sch_hfsc em_u32 cls_u32 cls_tcindex
[   94.047083]  cls_route cls_matchall cls_fw cls_flow cls_basic act_skbedit act_mirred act_gact ledtrig_usbport xt_set ip_set_list_set ip_set_hash_netportnet ip_set_hash_netport ip_set_hash_netnet ip_set_hash_netiface ip_set_hash_net ip_set_hash_mac ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_hash_ipmark ip_set_hash_ip ip_set_bitmap_port ip_set_bitmap_ipmac ip_set_bitmap_ip ip_set nfnetlink ip6table_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip6t_NPT ip6t_rt ip6t_mh ip6t_ipv6header ip6t_hbh ip6t_frag ip6t_eui64 ip6t_ah nf_log_ipv6 nf_log_common ip6table_mangle ip6table_filter ip6_tables ip6t_REJECT x_tables nf_reject_ipv6 nfsv4 nfsd nfs nfs_ssc ip6_udp_tunnel udp_tunnel sit netlink_diag tunnel4 ip_tunnel rpcsec_gss_krb5 auth_rpcgss oid_registry lockd sunrpc grace cifs dns_resolver dm_mirror dm_region_hash dm_log dm_crypt dm_mod dax nls_utf8 zram zsmalloc sha512_generic sha1_generic seqiv jitterentropy_rng drbg md5 md4 kpp hmac ecb des_generic libdes cts
[   94.134724]  cmac arc4 cbc sha256_generic libsha256 encrypted_keys trusted tpm usb_storage leds_gpio xhci_plat_hcd xhci_pci xhci_mtk xhci_hcd sd_mod scsi_mod gpio_button_hotplug ext4 mbcache jbd2 usbcore nls_base usb_common crc32c_generic [last unloaded: mt7603e]
[   94.245046] CPU: 2 PID: 21 Comm: ksoftirqd/2 Tainted: G      D           5.10.64 #0
[   94.252672] Stack : 80a70000 00000000 8093e310 80084ec4 808f0000 807d7d94 00000000 00000000
[   94.261044]         80c99b24 80a50000 8079effc 80c90fc8 8087eec7 00000001 80c99ac8 8f8604cd
[   94.269416]         00000000 00000000 8079effc 80c99968 ffffefff 00000000 ffffffea 00000000
[   94.277789]         80c99974 000001d1 80884a68 ffffffff 00000000 00000000 00000000 807a0000
[   94.286161]         00000009 00000040 3fffffff 00000000 00000018 804220e4 00000008 80a50008
[   94.294529]         ...
[   94.296977] Call Trace:
[   94.299440] [<800081e0>] show_stack+0x30/0x100
[   94.303891] [<8039a29c>] dump_stack+0x9c/0xcc
[   94.308263] [<80031640>] __warn+0xc0/0x12c
[   94.312353] [<80031708>] warn_slowpath_fmt+0x5c/0xac
[   94.317311] [<800c3938>] smp_call_function_many_cond+0x438/0x454
[   94.323304] [<800c3998>] smp_call_function+0x24/0x30
[   94.328260] [<800310b4>] panic+0x120/0x320
[   94.332350] [<800084c8>] die+0x134/0x13c
[   94.336265] [<800088c0>] do_trap_or_bp+0x150/0x1bc
[   94.341049] [<80008a04>] do_bp+0xd8/0x280
[   94.345051] [<80003f0c>] handle_bp_int+0x28/0x3c
[   94.349681] [<806f135c>] __do_softirq+0x164/0x368
[   94.354368] 
[   94.355853] ---[ end trace 9d515235432dc445 ]---
[   94.360474] Rebooting in 3 seconds..
bol-van commented 3 years ago

May be this is related to another problem. I'm still investigating it. Suddenly one core kernel cpu usage raises to 100% perf top shows

    45.60%  [kernel]       [k] tasklet_action_common.isra.16
    24.27%  [kernel]       [k] __softirqentry_text_start
     5.65%  [kernel]       [k] r4k_wait_irqoff
     3.38%  [kernel]       [k] rcu_qs
     3.32%  [kernel]       [k] tasklet_action
     2.56%  [kernel]       [k] run_ksoftirqd
     1.77%  [kernel]       [k] smpboot_thread_fn

In this condition I did module reload and run perf and kernel crashed most times. I couldn't capture dmesg log yet. Will do it when possible