search

tempesta-tech / tempesta

All-in-one solution for high performance web content delivery and advanced protection against DDoS and web attacks
https://tempesta-tech.com/
GNU General Public License v2.0
621 stars 103 forks source link

GPF in `skb_clone` #1937

Closed RomanBelozerov closed 1 year ago

RomanBelozerov commented 1 year ago

Scope

hash - 872f58a59de6f7c9140eca0321a955d38601f4e4

My config (Tempesta on separate VM) :

[General]
ip = 192.168.122.1
ipv6 = ::1
verbose = 6
workdir = /tmp/host
duration = 10
concurrent_connections = 10
log_file = tests_log.log
stress_threads = 2
stress_large_content_length = 65536
stress_requests_count = 100
stress_mtu = 1500
long_body_size = 500
[Client]
ip = 192.168.122.1
ipv6 = ::1
hostname = localhost
ab = ab
wrk = wrk
h2load = h2load
tls-perf = tls-perf
workdir = /tmp/client
unavaliable_timeout = 300

[Tempesta]
ip = 192.168.122.116
ipv6 = ::1
hostname = 192.168.122.116
user = root
port = 22
srcdir = /root/tempesta
workdir = /tmp/host
config = tempesta.conf
tmp_config = tempesta_tmp.conf
unavaliable_timeout = 300

[Server]
ip = 192.168.122.1
ipv6 = ::1
hostname = localhost
user = root
port = 22
nginx = nginx
workdir = /tmp/nginx
resources = /var/www/html/
aliases_interface = virbr0
aliases_base_ip = 192.168.122.1
max_workers = 16
keepalive_timeout = 60
keepalive_requests = 100
unavaliable_timeout = 300
[ 1903.037431] general protection fault, probably for non-canonical address 0x3a31ba7af582fb1c: 0000 [#1] SMP NOPTI
[ 1903.038134] CPU: 4 PID: 0 Comm: swapper/4 Tainted: G           OE     5.10.35+ #1
[ 1903.038603] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.15.0-1 04/01/2014
[ 1903.039122] RIP: 0010:skb_clone+0x8d/0x100
[ 1903.039389] Code: b1 02 00 65 48 03 05 7a 48 51 55 48 83 00 01 4c 89 e6 e8 36 cf ff ff 4c 8b 65 f8 c9 c3 48 8b 40 28 48 85 c0 74 a1 a8 01 75 09 <48> 81 38 c0 bf af aa 74 94 4c 89 e7 89 75 f4 e8 8f f5 ff ff 8b 75
[ 1903.040516] RSP: 0018:ffffae3900184ce0 EFLAGS: 00010246
[ 1903.040833] RAX: 3a31ba7af582fb1c RBX: ffff91d2e7f33658 RCX: 0000000000000a20
[ 1903.041263] RDX: 0000000000000e7d RSI: 0000000000000a20 RDI: ffff91d2e7f33600
[ 1903.041692] RBP: ffffae3900184cf0 R08: 00000000d1b5e3b4 R09: 00000000000008eb
[ 1903.042121] R10: ffff91d2e7f33600 R11: 0000000000000010 R12: ffff91d2e7f33600
[ 1903.042549] R13: ffff91d2e7f33600 R14: ffff91d2ea550000 R15: 0000000000000000
[ 1903.042993] FS:  0000000000000000(0000) GS:ffff91d437d00000(0000) knlGS:0000000000000000
[ 1903.043499] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 1903.043853] CR2: 00007f01cd33919c CR3: 00000001230c0005 CR4: 0000000000770ee0
[ 1903.044296] PKRU: 55555554
[ 1903.044469] Call Trace:
[ 1903.044628]  <IRQ>
[ 1903.044762]  __tcp_transmit_skb+0x612/0xcd0
[ 1903.045023]  ? tso_fragment+0x149/0x210
[ 1903.045272]  ? tfw_sk_write_xmit+0x6e/0xd0 [tempesta_fw]
[ 1903.045597]  tcp_write_xmit+0x3c5/0x1060
[ 1903.045841]  __tcp_push_pending_frames+0x37/0x100
[ 1903.046129]  tcp_push+0xd3/0x100
[ 1903.046332]  ss_tx_action+0x308/0x600 [tempesta_fw]
[ 1903.046629]  net_tx_action+0x9c/0x250
[ 1903.046877]  __do_softirq+0xe3/0x340
[ 1903.047108]  asm_call_irq_on_stack+0xf/0x20
[ 1903.047381]  </IRQ>
[ 1903.047520]  do_softirq_own_stack+0x3d/0x50
[ 1903.047778]  irq_exit_rcu+0xa2/0xe0
[ 1903.047992]  sysvec_call_function_single+0x3d/0x90
[ 1903.048280]  asm_sysvec_call_function_single+0x12/0x20
[ 1903.048587] RIP: 0010:native_safe_halt+0xe/0x10
[ 1903.048858] Code: 39 ff ff ff 4c 89 ee 48 c7 c7 a0 ba c5 ab e8 19 7b 8e ff e9 01 ff ff ff cc cc cc cc e9 07 00 00 00 0f 00 2d 66 a8 44 00 fb f4 <c3> 90 e9 07 00 00 00 0f 00 2d 56 a8 44 00 f4 c3 cc cc 0f 1f 44 00
[ 1903.049953] RSP: 0018:ffffae39000a3e88 EFLAGS: 00000216
[ 1903.050265] RAX: ffffffffaadc02c0 RBX: 0000000000000004 RCX: ffff91d437d2ce40
[ 1903.050689] RDX: 00000000000ad0ee RSI: 0000000000000082 RDI: 0000000000000082
[ 1903.051122] RBP: ffffae39000a3e90 R08: 000000cd42e4dffb R09: 000000000000002f
[ 1903.051551] R10: 000001bb1a1c847d R11: 0000000000000004 R12: ffff91d2e02e1e40
[ 1903.051971] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
[ 1903.052390]  ? __sched_text_end+0x4/0x4
[ 1903.052623]  ? default_idle+0xe/0x20
[ 1903.052842]  arch_cpu_idle+0x15/0x20
[ 1903.053060]  default_idle_call+0x3d/0xc0
[ 1903.053299]  do_idle+0x215/0x2a0
[ 1903.053508]  cpu_startup_entry+0x20/0x30
[ 1903.053786]  start_secondary+0x145/0x1b0
[ 1903.054053]  secondary_startup_64_no_verify+0xc2/0xcb
[ 1903.054374] Modules linked in: tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) sha256_ssse3 sha512_ssse3 xt_conntrack nft_chain_nat xt_MASQUERADE nf_nat nf_conntrack_netlink nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 xfrm_user xfrm_algo nft_counter xt_addrtype nft_compat br_netfilter bridge stp llc snd_hda_codec_generic ledtrig_audio snd_hda_intel snd_intel_dspcfg snd_hda_codec snd_hda_core intel_rapl_msr snd_hwdep intel_rapl_common snd_pcm overlay binfmt_misc kvm_intel qxl snd_seq_midi kvm drm_ttm_helper ttm snd_seq_midi_event snd_rawmidi drm_kms_helper snd_seq crct10dif_pclmul ghash_clmulni_intel aesni_intel snd_seq_device crypto_simd cec snd_timer cryptd glue_helper snd rc_core nls_iso8859_1 input_leds fb_sys_fops serio_raw joydev soundcore syscopyarea sysfillrect sysimgblt mac_hid qemu_fw_cfg sch_fq_codel msr drm parport_pc ppdev lp parport ramoops pstore_blk reed_solomon pstore_zone efi_pstore virtio_rng ip_tables x_tables autofs4 hid_generic usbhid hid psmouse ahci
[ 1903.054422]  xhci_pci virtio_net i2c_i801 crc32_pclmul libahci net_failover i2c_smbus lpc_ich virtio_blk failover xhci_pci_renesas [last unloaded: tempesta_lib]
[ 1903.060651] ---[ end trace aa0f6db97404a35a ]---
[ 1903.060946] RIP: 0010:skb_clone+0x8d/0x100
[ 1903.061207] Code: b1 02 00 65 48 03 05 7a 48 51 55 48 83 00 01 4c 89 e6 e8 36 cf ff ff 4c 8b 65 f8 c9 c3 48 8b 40 28 48 85 c0 74 a1 a8 01 75 09 <48> 81 38 c0 bf af aa 74 94 4c 89 e7 89 75 f4 e8 8f f5 ff ff 8b 75
[ 1903.062364] RSP: 0018:ffffae3900184ce0 EFLAGS: 00010246
[ 1903.062696] RAX: 3a31ba7af582fb1c RBX: ffff91d2e7f33658 RCX: 0000000000000a20
[ 1903.063143] RDX: 0000000000000e7d RSI: 0000000000000a20 RDI: ffff91d2e7f33600
[ 1903.063594] RBP: ffffae3900184cf0 R08: 00000000d1b5e3b4 R09: 00000000000008eb
[ 1903.064046] R10: ffff91d2e7f33600 R11: 0000000000000010 R12: ffff91d2e7f33600
[ 1903.064500] R13: ffff91d2e7f33600 R14: ffff91d2ea550000 R15: 0000000000000000
[ 1903.064952] FS:  0000000000000000(0000) GS:ffff91d437d00000(0000) knlGS:0000000000000000
[ 1903.065465] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 1903.065832] CR2: 00007f01cd33919c CR3: 00000001230c0005 CR4: 0000000000770ee0
[ 1903.066287] PKRU: 55555554
[ 1903.066465] Kernel panic - not syncing: Fatal exception in interrupt
[ 1903.067289] Kernel Offset: 0x29000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
[ 1903.068400] Rebooting in 1 seconds..
[ 1904.060897] ACPI MEMORY or I/O RESET_REG.

Testing

const-t commented 1 year ago

Seems like a #1915. Need to pay attention to current task during working on #1915.