search

tempesta-tech / tempesta

All-in-one solution for high performance web content delivery and advanced protection against DDoS and web attacks
https://tempesta-tech.com/
GNU General Public License v2.0
618 stars 103 forks source link

Crash on sending_302_without_preparing unit test #1157

Closed vankoven closed 5 years ago

vankoven commented 5 years ago

Run unit tests to reproduce the issue:

[   60.400368] tfw_test: TEST_SUITE_RUN(tls)
[   60.401809] tfw_test: TEST_RUN(tls, ecp)
[   60.414887] ------------[ cut here ]------------
[   60.416210] WARNING: CPU: 1 PID: 1002 at /home/user/kernel/linux-packages/arch/x86/kernel/fpu/core.c:104 __kernel_fpu_begin_bh+0x75/0x80
[   60.420037] Modules linked in: tfw_test(O+) tfw_fuzzer(O) tempesta_fw(O) tempesta_db(O) tempesta_tls(O) tempesta_lib(O) kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel iTCO_wdt iTCO_vendor_support snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core sg snd_hwdep snd_pcm qxl snd_timer evdev snd virtio_balloon virtio_console ttm soundcore binfmt_misc pcspkr lpc_ich serio_raw drm_kms_helper mfd_core shpchp drm button ip_tables x_tables autofs4 ext4 crc16 mbcache jbd2 crc32c_generic fscrypto ecb sr_mod cdrom crc32c_intel virtio_blk virtio_net aesni_intel aes_x86_64 ahci psmouse crypto_simd cryptd libahci glue_helper i2c_i801 ehci_pci libata uhci_hcd ehci_hcd usbcore usb_common sym53c8xx scsi_transport_spi scsi_mod virtio_pci virtio_ring virtio
[   60.444837] CPU: 1 PID: 1002 Comm: insmod Tainted: G           O    4.14.0-tempesta-amd64 #1 Debian 4.14.32-tfw3-1
[   60.447267] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.12.0-20181126_142135-anatol 04/01/2014
[   60.450696] task: ffff9932edcf5a00 task.stack: ffffbb1a005b4000
[   60.452055] RIP: 0010:__kernel_fpu_begin_bh+0x75/0x80
[   60.453230] RSP: 0018:ffffbb1a005b7a30 EFLAGS: 00010246
[   60.454346] RAX: 0000000000000000 RBX: ffff9932edcf5a00 RCX: 0000000000000000
[   60.456157] RDX: 0000000000000000 RSI: 0000000000000001 RDI: 0000000000000000
[   60.458239] RBP: 0000000000000001 R08: 0000000000000345 R09: ffffffff9a3f5e40
[   60.460177] R10: 0000000000000001 R11: ffffffff9a3f6760 R12: 0000000000000000
[   60.462063] R13: 0000000000000001 R14: ffffffffc10d2b00 R15: ffffffffc10d2b50
[   60.465043] FS:  00007f8b5d2c0540(0000) GS:ffff9932ffd00000(0000) knlGS:0000000000000000
[   60.467840] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   60.469530] CR2: 00007ffb1bbd2000 CR3: 000000006bf42004 CR4: 00000000003606e0
[   60.471017] Call Trace:
[   60.471712]  ttls_ecp_group_free+0x90/0xb0 [tempesta_tls]
[   60.473335]  ttls_ecp_group_load+0x15/0x7b0 [tempesta_tls]
[   60.474481]  ? vsnprintf+0xf3/0x4d0
[   60.475583]  ttls_ecp_self_test+0xb1/0x580 [tempesta_tls]
[   60.477565]  ? wait_for_xmitr+0x40/0xb0
[   60.478979]  ? ttls_ecp_check_pub_priv+0x170/0x170 [tempesta_tls]
[   60.480682]  test_suite__tls+0x5c/0x4e0 [tfw_test]
[   60.482261]  ? 0xffffffffc0a44000
[   60.483473]  test_run_all+0xb7/0x3a0 [tfw_test]
[   60.485126]  tfw_test_init+0x14/0x1000 [tfw_test]
[   60.486815]  ? 0xffffffffc0a44000
[   60.487762]  do_one_initcall+0x51/0x190
[   60.489171]  ? kmem_cache_alloc_trace+0x14f/0x1a0
[   60.490830]  ? do_init_module+0x22/0x1f8
[   60.492737]  do_init_module+0x5b/0x1f8
[   60.494281]  load_module+0x25fc/0x2d40
[   60.495815]  ? vfs_read+0x119/0x130
[   60.497117]  ? SYSC_finit_module+0xd2/0x100
[   60.498646]  SYSC_finit_module+0xd2/0x100
[   60.500090]  do_syscall_64+0x8d/0x120
[   60.501424]  entry_SYSCALL_64_after_hwframe+0x3d/0xa2
[   60.503202] RIP: 0033:0x7f8b5cde28f9
[   60.504523] RSP: 002b:00007fffbc3f67a8 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[   60.507154] RAX: ffffffffffffffda RBX: 00005591b81527b0 RCX: 00007f8b5cde28f9
[   60.509724] RDX: 0000000000000000 RSI: 00005591b79f1638 RDI: 0000000000000003
[   60.512238] RBP: 00005591b79f1638 R08: 0000000000000000 R09: 00007f8b5d0a8000
[   60.514917] R10: 0000000000000003 R11: 0000000000000246 R12: 0000000000000000
[   60.517212] R13: 00005591b8152760 R14: 0000000000000000 R15: 0000000000000000
[   60.519345] Code: c0 74 2c b8 ff ff ff ff 89 c2 48 0f ae 37 31 c0 85 c0 74 d7 0f 0b 5b c3 48 0f ae 83 c0 0b 00 00 5b c3 e8 2f ff ff ff 84 c0 75 a6 <0f> 0b eb a2 0f 0b eb d0 0f 1f 00 0f 1f 44 00 00 65 8b 05 b4 68 
[   60.525354] ---[ end trace 437fb0a2dbc39af0 ]---
[   60.527462] ------------[ cut here ]------------
[   60.529215] WARNING: CPU: 1 PID: 1002 at /home/user/kernel/linux-packages/arch/x86/kernel/fpu/core.c:46 kernel_fpu_disable+0x19/0x20
[   60.533353] Modules linked in: tfw_test(O+) tfw_fuzzer(O) tempesta_fw(O) tempesta_db(O) tempesta_tls(O) tempesta_lib(O) kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel iTCO_wdt iTCO_vendor_support snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core sg snd_hwdep snd_pcm qxl snd_timer evdev snd virtio_balloon virtio_console ttm soundcore binfmt_misc pcspkr lpc_ich serio_raw drm_kms_helper mfd_core shpchp drm button ip_tables x_tables autofs4 ext4 crc16 mbcache jbd2 crc32c_generic fscrypto ecb sr_mod cdrom crc32c_intel virtio_blk virtio_net aesni_intel aes_x86_64 ahci psmouse crypto_simd cryptd libahci glue_helper i2c_i801 ehci_pci libata uhci_hcd ehci_hcd usbcore usb_common sym53c8xx scsi_transport_spi scsi_mod virtio_pci virtio_ring virtio
[   60.557298] CPU: 1 PID: 1002 Comm: insmod Tainted: G        W  O    4.14.0-tempesta-amd64 #1 Debian 4.14.32-tfw3-1
[   60.561450] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.12.0-20181126_142135-anatol 04/01/2014
[   60.565074] task: ffff9932edcf5a00 task.stack: ffffbb1a005b4000
[   60.567204] RIP: 0010:kernel_fpu_disable+0x19/0x20
[   60.568859] RSP: 0018:ffffbb1a005b7a28 EFLAGS: 00010202
[   60.570766] RAX: 0000000000000001 RBX: ffff9932edcf5a00 RCX: 0000000000000000
[   60.573183] RDX: 0000000000000000 RSI: 0000000000000001 RDI: 0000000000000000
[   60.575744] RBP: 0000000000000001 R08: 0000000000000345 R09: ffffffff9a3f5e40
[   60.578311] R10: 0000000000000001 R11: ffffffff9a3f6760 R12: 0000000000000000
[   60.580878] R13: 0000000000000001 R14: ffffffffc10d2b00 R15: ffffffffc10d2b50
[   60.583456] FS:  00007f8b5d2c0540(0000) GS:ffff9932ffd00000(0000) knlGS:0000000000000000
[   60.587685] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   60.589779] CR2: 00007ffb1bbd2000 CR3: 000000006bf42004 CR4: 00000000003606e0
[   60.591235] Call Trace:
[   60.591639]  __kernel_fpu_begin_bh+0x20/0x80
[   60.592362]  ttls_ecp_group_free+0x90/0xb0 [tempesta_tls]
[   60.593703]  ttls_ecp_group_load+0x15/0x7b0 [tempesta_tls]
[   60.595084]  ? vsnprintf+0xf3/0x4d0
[   60.595752]  ttls_ecp_self_test+0xb1/0x580 [tempesta_tls]
[   60.597242]  ? wait_for_xmitr+0x40/0xb0
[   60.598163]  ? ttls_ecp_check_pub_priv+0x170/0x170 [tempesta_tls]
[   60.599712]  test_suite__tls+0x5c/0x4e0 [tfw_test]
[   60.600827]  ? 0xffffffffc0a44000
[   60.601848]  test_run_all+0xb7/0x3a0 [tfw_test]
[   60.603051]  tfw_test_init+0x14/0x1000 [tfw_test]
[   60.604531]  ? 0xffffffffc0a44000
[   60.605198]  do_one_initcall+0x51/0x190
[   60.606219]  ? kmem_cache_alloc_trace+0x14f/0x1a0
[   60.607482]  ? do_init_module+0x22/0x1f8
[   60.608668]  do_init_module+0x5b/0x1f8
[   60.609795]  load_module+0x25fc/0x2d40
[   60.610686]  ? vfs_read+0x119/0x130
[   60.611624]  ? SYSC_finit_module+0xd2/0x100
[   60.612907]  SYSC_finit_module+0xd2/0x100
[   60.613947]  do_syscall_64+0x8d/0x120
[   60.614792]  entry_SYSCALL_64_after_hwframe+0x3d/0xa2
[   60.615676] RIP: 0033:0x7f8b5cde28f9
[   60.616698] RSP: 002b:00007fffbc3f67a8 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[   60.618657] RAX: ffffffffffffffda RBX: 00005591b81527b0 RCX: 00007f8b5cde28f9
[   60.620879] RDX: 0000000000000000 RSI: 00005591b79f1638 RDI: 0000000000000003
[   60.622326] RBP: 00005591b79f1638 R08: 0000000000000000 R09: 00007f8b5d0a8000
[   60.624096] R10: 0000000000000003 R11: 0000000000000246 R12: 0000000000000000
[   60.626440] R13: 00005591b8152760 R14: 0000000000000000 R15: 0000000000000000
[   60.629006] Code: ee 14 12 00 48 89 df 5b 5d e9 54 5a 11 00 0f 1f 40 00 0f 1f 44 00 00 65 8a 05 ec 69 9e 66 84 c0 75 09 65 c6 05 e0 69 9e 66 01 c3 <0f> 0b eb f3 0f 1f 00 0f 1f 44 00 00 65 8a 05 cc 69 9e 66 84 c0 
[   60.635446] ---[ end trace 437fb0a2dbc39af1 ]---
[   60.637137]   ECP test #1 (constant op_count, base point G): 
[   60.640824] passed
[   60.643384]   ECP test #2 (constant op_count, other point): 
[   60.651711] passed
[   60.654306] 
[   60.654617] ------------[ cut here ]------------
[   60.655959] WARNING: CPU: 1 PID: 1002 at /home/user/kernel/linux-packages/arch/x86/kernel/fpu/core.c:52 kernel_fpu_enable+0x19/0x20
[   60.659238] Modules linked in: tfw_test(O+) tfw_fuzzer(O) tempesta_fw(O) tempesta_db(O) tempesta_tls(O) tempesta_lib(O) kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel iTCO_wdt iTCO_vendor_support snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core sg snd_hwdep snd_pcm qxl snd_timer evdev snd virtio_balloon virtio_console ttm soundcore binfmt_misc pcspkr lpc_ich serio_raw drm_kms_helper mfd_core shpchp drm button ip_tables x_tables autofs4 ext4 crc16 mbcache jbd2 crc32c_generic fscrypto ecb sr_mod cdrom crc32c_intel virtio_blk virtio_net aesni_intel aes_x86_64 ahci psmouse crypto_simd cryptd libahci glue_helper i2c_i801 ehci_pci libata uhci_hcd ehci_hcd usbcore usb_common sym53c8xx scsi_transport_spi scsi_mod virtio_pci virtio_ring virtio
[   60.681471] CPU: 1 PID: 1002 Comm: insmod Tainted: G        W  O    4.14.0-tempesta-amd64 #1 Debian 4.14.32-tfw3-1
[   60.685347] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.12.0-20181126_142135-anatol 04/01/2014
[   60.688899] task: ffff9932edcf5a00 task.stack: ffffbb1a005b4000
[   60.691124] RIP: 0010:kernel_fpu_enable+0x19/0x20
[   60.692839] RSP: 0018:ffffbb1a005b7c78 EFLAGS: 00010246
[   60.694831] RAX: 00000000ffffff00 RBX: 0000000000000000 RCX: ffff9932edcf5a00
[   60.697466] RDX: 00000000ffffffff RSI: 0000000000000086 RDI: ffff9932edcf65c0
[   60.700071] RBP: ffffbb1a005b7e70 R08: 000000000009fbe2 R09: 0000000000000311
[   60.702765] R10: ffffbb1a005b79c0 R11: ffffffff9b38ae6d R12: 0000000000000000
[   60.705092] R13: 0000000000000001 R14: ffffffffc10d2b00 R15: ffffffffc10d2b50
[   60.707250] FS:  00007f8b5d2c0540(0000) GS:ffff9932ffd00000(0000) knlGS:0000000000000000
[   60.709313] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   60.710714] CR2: 00007ffb1bbd2000 CR3: 000000006bf42004 CR4: 00000000003606e0
[   60.713052] Call Trace:
[   60.713857]  __kernel_fpu_end_bh+0x30/0x80
[   60.715058]  kernel_fpu_end+0x27/0x30
[   60.716039]  test_suite__tls+0x6b/0x4e0 [tfw_test]
[   60.717232]  ? 0xffffffffc0a44000
[   60.718090]  test_run_all+0xb7/0x3a0 [tfw_test]
[   60.719283]  tfw_test_init+0x14/0x1000 [tfw_test]
[   60.720683]  ? 0xffffffffc0a44000
[   60.721416]  do_one_initcall+0x51/0x190
[   60.722380]  ? kmem_cache_alloc_trace+0x14f/0x1a0
[   60.723630]  ? do_init_module+0x22/0x1f8
[   60.724643]  do_init_module+0x5b/0x1f8
[   60.725635]  load_module+0x25fc/0x2d40
[   60.726370]  ? vfs_read+0x119/0x130
[   60.727259]  ? SYSC_finit_module+0xd2/0x100
[   60.728327]  SYSC_finit_module+0xd2/0x100
[   60.729334]  do_syscall_64+0x8d/0x120
[   60.730253]  entry_SYSCALL_64_after_hwframe+0x3d/0xa2
[   60.731566] RIP: 0033:0x7f8b5cde28f9
[   60.732462] RSP: 002b:00007fffbc3f67a8 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[   60.734435] RAX: ffffffffffffffda RBX: 00005591b81527b0 RCX: 00007f8b5cde28f9
[   60.736365] RDX: 0000000000000000 RSI: 00005591b79f1638 RDI: 0000000000000003
[   60.737937] RBP: 00005591b79f1638 R08: 0000000000000000 R09: 00007f8b5d0a8000
[   60.739862] R10: 0000000000000003 R11: 0000000000000246 R12: 0000000000000000
[   60.741340] R13: 00005591b8152760 R14: 0000000000000000 R15: 0000000000000000
[   60.743107] Code: 75 09 65 c6 05 e0 69 9e 66 01 c3 0f 0b eb f3 0f 1f 00 0f 1f 44 00 00 65 8a 05 cc 69 9e 66 84 c0 74 09 65 c6 05 c0 69 9e 66 00 c3 <0f> 0b eb f3 0f 1f 00 0f 1f 44 00 00 8b 15 ad d7 d3 01 53 31 f6 
[   60.748798] ---[ end trace 437fb0a2dbc39af2 ]---
[   60.750257] tfw_test: TEST_RUN(tls, mpi)
krizhanovsky commented 5 years ago

Unit tests are broken: just hit following crash on simple make clean test on current master:

[  436.908125] tfw_test: TEST_RUN(http_sticky, sending_302_without_preparing)
[  436.910569] BUG: spinlock wrong owner on CPU#0, insmod/4314
[  436.912457] kasan: CONFIG_KASAN_INLINE enabled
[  436.913960] kasan: GPF could be caused by NULL-ptr deref or user memory access
[  436.913966] general protection fault: 0000 [#1] SMP KASAN
[  436.913967] Modules linked in: tfw_test(O+) tfw_fuzzer(O) tempesta_fw(O) tempesta_db(O) tempesta_tls(O) tempesta_lib(O) binfmt_misc bochs_drm ttm drm_kms_helper drm ppdev crct10dif_pclmul parport_pc ghash_clmulni_intel fb_sys_fops parport syscopyarea sg serio_raw sysfillrect sysimgblt button pcspkr ip_tables x_tables ext4 crc16 mbcache jbd2 fscrypto sr_mod sd_mod cdrom ata_generic ata_piix libata scsi_mod e1000 psmouse i2c_piix4
[  436.914022] CPU: 0 PID: 4314 Comm: insmod Tainted: G        W  O    4.14.32-kdump+ #114
[  436.914023] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20180724_192412-buildhw-07.phx2.fedoraproject.org-1.fc29 04/01/2014
[  436.914024] task: ffff880116bce080 task.stack: ffff880106788000
[  436.914029] RIP: 0010:spin_dump+0xe3/0x280
[  436.914030] RSP: 0018:ffff88010678f298 EFLAGS: 00010202
[  436.914032] RAX: dffffc0000000000 RBX: ffffffffc16c5ef8 RCX: 0000000000000004
[  436.914033] RDX: 0000000022d79c92 RSI: ffff88011b41e5b8 RDI: 0000000116bce490
[  436.914034] RBP: 0000000116bce080 R08: 0000000000000000 R09: 0000000000000000
[  436.914035] R10: 0000000000016033 R11: ffffc900026ff020 R12: ffff880116bce080
[  436.914036] R13: ffff88010678f340 R14: dffffc0000000000 R15: ffff88010678f340
[  436.914038] FS:  00007fc7e34d3700(0000) GS:ffff88011b400000(0000) knlGS:0000000000000000
[  436.914039] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  436.914046] CR2: 00007f398c02f0f8 CR3: 000000010ab50002 CR4: 00000000003606f0
[  436.914049] Call Trace:
[  436.914069]  ? tfw_http_msg_free+0x1c6/0x2d0 [tfw_test]
[  436.914072]  do_raw_spin_unlock+0x173/0x240
[  436.914075]  _raw_spin_unlock_bh+0x5/0x20
[  436.914099]  tfw_http_resp_fwd+0x588/0xc50 [tfw_test]
[  436.914114]  ? tfw_http_conn_resend+0x1b10/0x1b10 [tfw_test]
[  436.914117]  ? _raw_spin_unlock+0x5/0x10
[  436.914131]  test_suite__http_sticky+0x486/0x47b0 [tfw_test]
[  436.914144]  ? test_helper_sticky_stop+0x30/0x30 [tfw_test]
[  436.914147]  ? soft_cursor+0x5bd/0xc40
[  436.914149]  ? bit_clear+0x540/0x540
[  436.914153]  ? get_color+0x2a/0x3a0
[  436.914167]  ? test_suite__addr+0xf0/0xf0 [tfw_test]
[  436.914169]  ? printk+0x97/0xbe
[  436.914171]  ? show_regs_print_info+0x53/0x53
[  436.914185]  ? tfw_http_msg_free+0x1c6/0x2d0 [tfw_test]
[  436.914187]  ? 0xffffffffc1ee8000
[  436.914201]  test_run_all+0x230/0x390 [tfw_test]
[  436.914203]  ? 0xffffffffc1ee8000
[  436.914216]  tfw_test_init+0x14/0x1000 [tfw_test]
[  436.914218]  do_one_initcall+0x90/0x210
[  436.914220]  ? initcall_blacklisted+0x150/0x150
[  436.914221]  ? kasan_kmalloc+0xa0/0xd0
[  436.914224]  ? kmem_cache_alloc_trace+0xd1/0x570
krizhanovsky commented 5 years ago

The last Oops is reproduced on https://github.com/tempesta-tech/tempesta/pull/1158 , so assign to @ikoveshnikov : probably the problem is already fixed in some recent PRs.

krizhanovsky commented 5 years ago

Test crash https://github.com/tempesta-tech/tempesta/issues/1157#issuecomment-457836042 is still here

vankoven commented 5 years ago

I couldn't reproduced the crash on unit tests neither with KASAN disabled nor with enabled. But there are some KASAN warnings and crashes.

Start Tempesta with blank configuration:

[   57.028660] tempesta_lib: loading out-of-tree module taints kernel.
[   57.187171] ------------[ cut here ]------------
[   57.188180] WARNING: CPU: 1 PID: 1569 at kernel/module.c:1133 module_put+0x1a4/0x1c0
[   57.190752] Modules linked in: ctr gcm tempesta_tls(O+) tempesta_lib(O) kvm_intel iTCO_wdt iTCO_vendor_support kvm irqbypass crct10dif_pclmul crc32_pclmul qxl ghash_clmulni_intel cryptd ttm virtio_console drm_kms_helper sg virtio_balloon evdev serio_raw pcspkr drm lpc_ich mfd_core shpchp binfmt_misc button ip_tables x_tables autofs4 ext4 crc16 mbcache jbd2 fscrypto sr_mod cdrom virtio_net virtio_blk crc32c_intel ehci_pci uhci_hcd ehci_hcd ahci libahci psmouse sym53c8xx scsi_transport_spi libata virtio_pci usbcore i2c_i801 virtio_ring virtio scsi_mod
[   57.205743] CPU: 1 PID: 1569 Comm: insmod Tainted: G           O    4.14.32+ #1
[   57.207678] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.12.0-20181126_142135-anatol 04/01/2014
[   57.211727] task: ffff88000ab7b280 task.stack: ffff880038290000
[   57.213137] RIP: 0010:module_put+0x1a4/0x1c0
[   57.214764] RSP: 0018:ffff880038297508 EFLAGS: 00010297
[   57.216877] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 00000000ffffffff
[   57.218824] RDX: 0000000000000000 RSI: dffffc0000000000 RDI: ffff880038297528
[   57.220414] RBP: 1ffff10007052ea1 R08: 0000000088d55c0b R09: fffffbfff5b3a443
[   57.222216] R10: 0000000000000001 R11: fffffbfff5b3a442 R12: ffffffffc05b6340
[   57.224906] R13: 1ffff10007052ea5 R14: ffffffffc05b6668 R15: dffffc0000000000
[   57.226582] FS:  00007f7c3ff1b540(0000) GS:ffff88006cb00000(0000) knlGS:0000000000000000
[   57.229908] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   57.231272] CR2: 000055919478f280 CR3: 00000000664b4006 CR4: 00000000003606e0
[   57.232975] Call Trace:
[   57.233448]  ? find_module+0x20/0x20
[   57.234869]  ? __crypto_alloc_tfm+0x10f/0x210
[   57.236075]  ? kasan_slab_free+0x88/0xc0
[   57.237927]  ? crypto_destroy_tfm+0xd4/0x130
[   57.239129]  crypto_destroy_tfm+0xa2/0x130
[   57.240467]  crypto_destroy_tfm+0xd4/0x130
[   57.241628]  ? crypto_aead_setauthsize+0x70/0x70
[   57.243194]  crypto_gcm_exit_tfm+0x24/0x40 [gcm]
[   57.243995]  crypto_destroy_tfm+0x61/0x130
[   57.245976]  ttls_crypto_modinit+0xa5/0x221 [tempesta_tls]
[   57.248167]  ? ttls_mpi_modinit+0x2d/0xcbb [tempesta_tls]
[   57.250545]  ? 0xffffffffc06d8000
[   57.252386]  ttls_init+0x1d/0xda [tempesta_tls]
[   57.254307]  ? 0xffffffffc06d8000
[   57.255272]  do_one_initcall+0x91/0x203
[   57.256041]  ? initcall_blacklisted+0x190/0x190
[   57.257420]  ? kasan_unpoison_shadow+0x30/0x40
[   57.258826]  ? kasan_kmalloc+0xa0/0xd0
[   57.260377]  ? do_init_module+0x86/0x3b6
[   57.261331]  ? kmem_cache_alloc_trace+0x11d/0x270
[   57.262463]  ? kasan_unpoison_shadow+0x30/0x40
[   57.263975]  ? __asan_register_globals+0x6e/0x80
[   57.265324]  do_init_module+0x13a/0x3b6
[   57.266748]  ? kzalloc.constprop.53+0xa/0xa
[   57.268421]  ? kasan_slab_free+0x88/0xc0
[   57.269116]  ? load_module+0x3eb5/0x4c00
[   57.269870]  load_module+0x3ec6/0x4c00
[   57.270833]  ? module_frob_arch_sections+0x20/0x20
[   57.272220]  ? __vfs_read+0x2dd/0x380
[   57.273990]  ? vfs_copy_file_range+0x450/0x450
[   57.275495]  ? vmap_page_range_noflush+0x2e7/0x3e0
[   57.277453]  ? __fsnotify_update_child_dentry_flags.part.1+0x160/0x160
[   57.280054]  ? __fsnotify_update_child_dentry_flags.part.1+0x160/0x160
[   57.282689]  ? kernel_read+0x74/0xa0
[   57.283356]  ? kernel_read_file+0x163/0x300
[   57.284466]  ? open_exec+0x40/0x40
[   57.285265]  ? up_read+0x20/0x20
[   57.286186]  ? SYSC_finit_module+0x1db/0x200
[   57.287394]  SYSC_finit_module+0x1db/0x200
[   57.289100]  ? SYSC_init_module+0x240/0x240
[   57.290935]  ? SYSC_newfstat+0x7c/0xd0
[   57.292353]  ? SyS_init_module+0x10/0x10
[   57.294056]  do_syscall_64+0x17d/0x320
[   57.295769]  ? syscall_return_slowpath+0x1c0/0x1c0
[   57.297930]  ? async_page_fault+0x2f/0x50
[   57.299668]  ? do_page_fault+0x90/0x220
[   57.301377]  ? __do_page_fault+0x6d0/0x6d0
[   57.302301]  ? prepare_exit_to_usermode+0xde/0x140
[   57.303946]  ? perf_trace_sys_enter+0x490/0x490
[   57.305774]  ? __put_user_4+0x1c/0x30
[   57.306664]  entry_SYSCALL_64_after_hwframe+0x3d/0xa2
[   57.308733] RIP: 0033:0x7f7c3fa3d8f9
[   57.310240] RSP: 002b:00007ffc6feadfb8 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[   57.313004] RAX: ffffffffffffffda RBX: 000055b380384790 RCX: 00007f7c3fa3d8f9
[   57.315843] RDX: 0000000000000000 RSI: 000055b37fb14638 RDI: 0000000000000004
[   57.318699] RBP: 000055b37fb14638 R08: 0000000000000000 R09: 00007f7c3fd03000
[   57.321469] R10: 0000000000000004 R11: 0000000000000246 R12: 0000000000000000
[   57.324259] R13: 000055b380384750 R14: 0000000000000000 R15: 0000000000000000
[   57.327544] Code: c3 18 e8 d0 f7 1f 00 48 8b 7b f0 4c 89 f2 4c 89 e6 e8 81 1d e2 00 48 89 df e8 b9 f7 1f 00 4c 8b 2b 4d 85 ed 75 d4 e9 33 ff ff ff <0f> 0b e9 1f ff ff ff 89 c2 e9 fe fe ff ff e8 49 bb ee ff 66 0f 
[   57.335193] ---[ end trace 4eb733d16a870665 ]---
[   57.337507] ------------[ cut here ]------------
[   57.338334] WARNING: CPU: 1 PID: 1569 at kernel/module.c:1133 module_put+0x1a4/0x1c0
[   57.339616] Modules linked in: ctr gcm tempesta_tls(O+) tempesta_lib(O) kvm_intel iTCO_wdt iTCO_vendor_support kvm irqbypass crct10dif_pclmul crc32_pclmul qxl ghash_clmulni_intel cryptd ttm virtio_console drm_kms_helper sg virtio_balloon evdev serio_raw pcspkr drm lpc_ich mfd_core shpchp binfmt_misc button ip_tables x_tables autofs4 ext4 crc16 mbcache jbd2 fscrypto sr_mod cdrom virtio_net virtio_blk crc32c_intel ehci_pci uhci_hcd ehci_hcd ahci libahci psmouse sym53c8xx scsi_transport_spi libata virtio_pci usbcore i2c_i801 virtio_ring virtio scsi_mod
[   57.354910] CPU: 1 PID: 1569 Comm: insmod Tainted: G        W  O    4.14.32+ #1
[   57.356598] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.12.0-20181126_142135-anatol 04/01/2014
[   57.363300] task: ffff88000ab7b280 task.stack: ffff880038290000
[   57.365978] RIP: 0010:module_put+0x1a4/0x1c0
[   57.367920] RSP: 0018:ffff880038297508 EFLAGS: 00010297
[   57.369926] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 00000000ffffffff
[   57.373366] RDX: 0000000000000000 RSI: dffffc0000000000 RDI: ffff880038297528
[   57.375594] RBP: 1ffff10007052ea1 R08: 0000000088d55c0b R09: fffffbfff5b3a443
[   57.377750] R10: 0000000000000001 R11: fffffbfff5b3a442 R12: ffffffffc05b6340
[   57.380785] R13: 1ffff10007052ea5 R14: ffffffffc05b6668 R15: dffffc0000000000
[   57.382473] FS:  00007f7c3ff1b540(0000) GS:ffff88006cb00000(0000) knlGS:0000000000000000
[   57.384392] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   57.385739] CR2: 000055919478f280 CR3: 00000000664b4001 CR4: 00000000003606e0
[   57.387508] Call Trace:
[   57.388153]  ? find_module+0x20/0x20
[   57.388974]  ? __crypto_alloc_tfm+0x10f/0x210
[   57.390021]  ? kasan_slab_free+0x88/0xc0
[   57.391647]  ? crypto_destroy_tfm+0xd4/0x130
[   57.393637]  crypto_destroy_tfm+0xa2/0x130
[   57.394595]  crypto_destroy_tfm+0xd4/0x130
[   57.395913]  ? crypto_aead_setauthsize+0x70/0x70
[   57.397025]  crypto_gcm_exit_tfm+0x24/0x40 [gcm]
[   57.398458]  crypto_destroy_tfm+0x61/0x130
[   57.399487]  ttls_crypto_modinit+0xa5/0x221 [tempesta_tls]
[   57.400866]  ? ttls_mpi_modinit+0x2d/0xcbb [tempesta_tls]
[   57.402197]  ? 0xffffffffc06d8000
[   57.402796]  ttls_init+0x1d/0xda [tempesta_tls]
[   57.404184]  ? 0xffffffffc06d8000
[   57.404828]  do_one_initcall+0x91/0x203
[   57.405585]  ? initcall_blacklisted+0x190/0x190
[   57.407082]  ? kasan_unpoison_shadow+0x30/0x40
[   57.408331]  ? kasan_kmalloc+0xa0/0xd0
[   57.408990]  ? do_init_module+0x86/0x3b6
[   57.409665]  ? kmem_cache_alloc_trace+0x11d/0x270
[   57.411609]  ? kasan_unpoison_shadow+0x30/0x40
[   57.412866]  ? __asan_register_globals+0x6e/0x80
[   57.413866]  do_init_module+0x13a/0x3b6
[   57.415243]  ? kzalloc.constprop.53+0xa/0xa
[   57.415977]  ? kasan_slab_free+0x88/0xc0
[   57.416922]  ? load_module+0x3eb5/0x4c00
[   57.418548]  load_module+0x3ec6/0x4c00
[   57.419469]  ? module_frob_arch_sections+0x20/0x20
[   57.420652]  ? __vfs_read+0x2dd/0x380
[   57.421544]  ? vfs_copy_file_range+0x450/0x450
[   57.422631]  ? vmap_page_range_noflush+0x2e7/0x3e0
[   57.423793]  ? __fsnotify_update_child_dentry_flags.part.1+0x160/0x160
[   57.426441]  ? __fsnotify_update_child_dentry_flags.part.1+0x160/0x160
[   57.428211]  ? kernel_read+0x74/0xa0
[   57.429685]  ? kernel_read_file+0x163/0x300
[   57.431400]  ? open_exec+0x40/0x40
[   57.432226]  ? up_read+0x20/0x20
[   57.433464]  ? SYSC_finit_module+0x1db/0x200
[   57.434881]  SYSC_finit_module+0x1db/0x200
[   57.435860]  ? SYSC_init_module+0x240/0x240
[   57.437597]  ? SYSC_newfstat+0x7c/0xd0
[   57.438624]  ? SyS_init_module+0x10/0x10
[   57.440205]  do_syscall_64+0x17d/0x320
[   57.441733]  ? syscall_return_slowpath+0x1c0/0x1c0
[   57.442895]  ? async_page_fault+0x2f/0x50
[   57.443871]  ? do_page_fault+0x90/0x220
[   57.444938]  ? __do_page_fault+0x6d0/0x6d0
[   57.446375]  ? prepare_exit_to_usermode+0xde/0x140
[   57.447877]  ? perf_trace_sys_enter+0x490/0x490
[   57.448916]  ? __put_user_4+0x1c/0x30
[   57.449854]  entry_SYSCALL_64_after_hwframe+0x3d/0xa2
[   57.451062] RIP: 0033:0x7f7c3fa3d8f9
[   57.452539] RSP: 002b:00007ffc6feadfb8 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[   57.454335] RAX: ffffffffffffffda RBX: 000055b380384790 RCX: 00007f7c3fa3d8f9
[   57.456034] RDX: 0000000000000000 RSI: 000055b37fb14638 RDI: 0000000000000004
[   57.457833] RBP: 000055b37fb14638 R08: 0000000000000000 R09: 00007f7c3fd03000
[   57.459808] R10: 0000000000000004 R11: 0000000000000246 R12: 0000000000000000
[   57.461520] R13: 000055b380384750 R14: 0000000000000000 R15: 0000000000000000
[   57.463224] Code: c3 18 e8 d0 f7 1f 00 48 8b 7b f0 4c 89 f2 4c 89 e6 e8 81 1d e2 00 48 89 df e8 b9 f7 1f 00 4c 8b 2b 4d 85 ed 75 d4 e9 33 ff ff ff <0f> 0b e9 1f ff ff ff 89 c2 e9 fe fe ff ff e8 49 bb ee ff 66 0f 
[   57.467725] ---[ end trace 4eb733d16a870666 ]---
[   57.638976] [tdb] Start Tempesta DB
[   57.722075] [tempesta fw] Initializing Tempesta FW kernel module...

Send a request and get a crash:

[  136.221262] [tempesta fw]   new client: cli=ffff880066c92048
[  136.224071] [tempesta fw]   client address: 192.168.122.1
[  136.227399] [tempesta fw]     client ffff880066c92048, conn_users=1
[  136.229126] [tempesta fw]     client ffff880066c92048, conn_users=2
[  136.230751] [tempesta fw]     tfw_http_conn_init: conn=[ffff880065976d58]
[  136.233472] [tempesta fw]     Link new msg ffff88006666b020 with connection ffff880065976d58
[  136.236220] [tempesta fw]     Add skb ffff880065bbe7c0 to message ffff88006666b020
[  136.239270] [tempesta fw]     Received 83 client data bytes on conn=ffff880065976d58 msg=ffff88006666b020
[  136.245207] [tempesta fw]     Request parsed: len=83 next=ffff880065bbe7c0 parsed=83 msg_len=83 ver=3 res=0
[  136.251043] general protection fault: 0000 [#1] SMP KASAN PTI
[  136.253559] Modules linked in: tempesta_fw(O) tempesta_db(O) sha256_ssse3 sha512_ssse3 sha512_generic ccm ctr gcm tempesta_tls(O) tempesta_lib(O) kvm_intel iTCO_wdt iTCO_vendor_support kvm irqbypass crct10dif_pclmul crc32_pclmul qxl ghash_clmulni_intel cryptd ttm virtio_console drm_kms_helper sg virtio_balloon evdev serio_raw pcspkr drm lpc_ich mfd_core shpchp binfmt_misc button ip_tables x_tables autofs4 ext4 crc16 mbcache jbd2 fscrypto sr_mod cdrom virtio_net virtio_blk crc32c_intel ehci_pci uhci_hcd ehci_hcd ahci libahci psmouse sym53c8xx scsi_transport_spi libata virtio_pci usbcore i2c_i801 virtio_ring virtio scsi_mod
[  136.280072] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G        W  O    4.14.32+ #1
[  136.283328] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.12.0-20181126_142135-anatol 04/01/2014
[  136.287639] task: ffffffffad218bc0 task.stack: ffffffffad200000
[  136.290269] RIP: 0010:tfw_http_sess_obtain+0xe3/0x690 [tempesta_fw]
[  136.293072] RSP: 0018:ffff88006ca06088 EFLAGS: 00010246
[  136.295362] RAX: 0000000000000004 RBX: 1ffff1000d940c19 RCX: ffffffffc0cf27c1
[  136.298413] RDX: 0000000000000001 RSI: 000000000000001c RDI: ffff88006ca061e8
[  136.301409] RBP: ffff88006ca063a8 R08: ffffed000d940c40 R09: ffffed000d940c41
[  136.306327] R10: 0000000000000004 R11: ffffed000d940c40 R12: 0000000000000000
[  136.309860] R13: ffff88006ca061e8 R14: ffff88006666b020 R15: 0000000000000000
[  136.312424] FS:  0000000000000000(0000) GS:ffff88006ca00000(0000) knlGS:0000000000000000
[  136.315932] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  136.318070] CR2: 00007fbc708558a0 CR3: 0000000061c0e005 CR4: 00000000003606f0
[  136.321213] Call Trace:
[  136.322355]  <IRQ>
[  136.323287]  ? tfw_http_parse_req+0xa2a/0x16b60 [tempesta_fw]
[  136.325774]  ? dynamic_emit_prefix+0x27/0x210
[  136.327720]  ? tfw_http_sess_mark_name+0x10/0x10 [tempesta_fw]
[  136.330222]  ? __dynamic_pr_debug+0x123/0x180
[  136.331326]  ? dynamic_emit_prefix+0x210/0x210
[  136.332587]  ? tfw_http_msg_pair+0x80/0x80 [tempesta_fw]
[  136.335868]  ? ss_skb_process+0x354/0x3c0 [tempesta_fw]
[  136.338530]  ? ss_skb_chop_head_tail+0xe7/0x330 [tempesta_fw]
[  136.341407]  ? ss_skb_expand_head_tail+0x2a0/0x2a0 [tempesta_fw]
[  136.344078]  ? tfw_msg_iter_setup+0x80/0x80 [tempesta_fw]
[  136.346500]  tfw_http_msg_process+0xb73/0x1950 [tempesta_fw]
[  136.348979]  ? depot_save_stack+0x3b7/0x480
[  136.350320]  ? tfw_http_conn_drop+0x480/0x480 [tempesta_fw]
[  136.352483]  ? save_stack+0x89/0xb0
[  136.354007]  ? save_stack+0x32/0xb0
[  136.355545]  ? kasan_slab_free+0x72/0xc0
[  136.357253]  ? kmem_cache_free+0x98/0x220
[  136.358904]  ? kfree_skbmem+0x2ac/0x2f0
[  136.359843]  ? tcp_data_queue+0x83b/0x1a00
[  136.360877]  ? tcp_rcv_state_process+0x78e/0x1d90
[  136.362425]  ? tcp_child_process+0x17e/0x330
[  136.363405]  ? tcp_v4_rcv+0xc0c/0x1280
[  136.364131]  ? ip_local_deliver_finish+0x138/0x410
[  136.365303]  ? ip_local_deliver+0x109/0x270
[  136.367188]  ? ip_rcv_finish+0x292/0xab0
[  136.368143]  ? ip_rcv+0x516/0x7a0
[  136.369278]  ? __netif_receive_skb_core+0x103f/0x18b0
[  136.371479]  ? process_backlog+0x124/0x2a0
[  136.373290]  ? net_rx_action+0x4db/0xba0
[  136.375010]  ? __do_softirq+0x1a1/0x4a8
[  136.376388]  ? irq_exit+0xe1/0xf0
[  136.377044]  ? smp_call_function_single_interrupt+0xc0/0x230
[  136.378611]  ? call_function_single_interrupt+0x7d/0x90
[  136.380177]  ? native_safe_halt+0x2/0x10
[  136.381013]  ? default_idle+0x82/0x250
[  136.381742]  ? do_idle+0x1a6/0x220
[  136.382769]  ? cpu_startup_entry+0xdb/0xe0
[  136.383849]  ? start_kernel+0x57c/0x5bd
[  136.384473]  ? secondary_startup_64+0xa5/0xb0
[  136.385175]  ? vprintk_emit+0x40b/0x460
[  136.385797]  ? debug_object_activate+0x22f/0x390
[  136.386516]  ? kmsg_dump_rewind+0xa0/0xa0
[  136.387195]  ? tfw_http_conn_drop+0x480/0x480 [tempesta_fw]
[  136.388098]  __gfsm_fsm_exec+0xb4/0x120 [tempesta_fw]
[  136.388919]  ? kmem_cache_free+0x98/0x220
[  136.389585]  tfw_connection_recv+0xb5/0xf0 [tempesta_fw]
[  136.390447]  ? tfw_connection_send+0x60/0x60 [tempesta_fw]
[  136.391368]  ? skb_tx_error+0x100/0x100
[  136.392096]  ? mod_timer_pending+0x8b0/0x8b0
[  136.393754]  ss_tcp_process_data+0x4a5/0x920 [tempesta_fw]
[  136.397190]  ? ss_wq_push+0xb0/0xb0 [tempesta_fw]
[  136.399425]  ? tcpm_check_stamp+0x8c/0xe0
[  136.401188]  ? tcpm_suck_dst+0x1a0/0x1a0
[  136.402930]  ? tcp_get_metrics+0x224/0x630
[  136.404335]  ss_tcp_data_ready+0x111/0x190 [tempesta_fw]
[  136.406421]  ? ss_linkerror+0xa0/0xa0 [tempesta_fw]
[  136.408555]  ? tcp_event_data_recv+0x245/0x5c0
[  136.410478]  tcp_data_queue+0xf50/0x1a00
[  136.412245]  ? tcp_fin+0x350/0x350
[  136.413735]  ? tcp_fastretrans_alert+0xec0/0xec0
[  136.415770]  ? tcp_rcv_state_process+0x7ff/0x1d90
[  136.417653]  ? tfw_classify_tcp+0xbb/0x110 [tempesta_fw]
[  136.420015]  ? tfw_classify_conn_close+0xe0/0xe0 [tempesta_fw]
[  136.422536]  ? inet_csk_reqsk_queue_add+0x27c/0x2b0
[  136.423904]  ? pvclock_read_flags+0x50/0x50
[  136.426316]  ? tcp_urg+0x93/0x430
[  136.428631]  ? tcp_xmit_recovery.part.50+0x70/0x70
[  136.430845]  ? kvm_sched_clock_read+0x1a/0x30
[  136.432505]  ? sched_clock+0x5/0x10
[  136.433301]  tcp_rcv_established+0x416/0x9d0
[  136.434672]  ? tcp_data_queue+0x1a00/0x1a00
[  136.435875]  ? rt_cpu_seq_start+0x90/0xb0
[  136.436934]  ? tcp_filter+0xa0/0xa0
[  136.437575]  tcp_v4_do_rcv+0x129/0x340
[  136.438513]  tcp_v4_rcv+0x119a/0x1280
[  136.439789]  ? __inet_lookup_established+0x2ed/0x3f0
[  136.440763]  ? tcp_v4_early_demux+0x3a0/0x3a0
[  136.442140]  ip_local_deliver_finish+0x138/0x410
[  136.442949]  ? ip_rcv_finish+0xab0/0xab0
[  136.444348]  ? tdb_htrie_descend+0x64/0x170 [tempesta_db]
[  136.445549]  ip_local_deliver+0x109/0x270
[  136.446573]  ? ip_call_ra_chain+0x300/0x300
[  136.447332]  ? tcp_v4_early_demux+0x331/0x3a0
[  136.448112]  ? tcp_v4_send_synack+0x1c0/0x1c0
[  136.449167]  ? tdb_rec_get+0xd6/0x120 [tempesta_db]
[  136.450699]  ip_rcv_finish+0x292/0xab0
[  136.451429]  ? inet_del_offload+0x40/0x40
[  136.452195]  ? tfw_classify_shrink+0x10/0x10 [tempesta_fw]
[  136.453125]  ? deref_stack_reg+0xa1/0xe0
[  136.454262]  ? __save_stack_trace+0x5e/0x100
[  136.456083]  ? tcp_v4_send_synack+0x1c0/0x1c0
[  136.458362]  ? tfw_ipv6_nf_hook+0x270/0x270 [tempesta_fw]
[  136.460190]  ? __save_stack_trace+0x5e/0x100
[  136.461576]  ? stack_access_ok+0x35/0x80
[  136.462333]  ? tfw_ipv6_nf_hook+0x270/0x270 [tempesta_fw]
[  136.463311]  ? nf_hook_slow+0xa4/0xe0
[  136.464599]  ip_rcv+0x516/0x7a0
[  136.465182]  ? ip_local_deliver+0x270/0x270
[  136.465991]  ? inet_del_offload+0x40/0x40
[  136.467411]  ? _find_next_bit+0x73/0xb0
[  136.468507]  ? cpumask_next_and+0x43/0x60
[  136.469909]  ? update_sd_lb_stats+0xbbc/0xc60
[  136.470755]  ? ip_local_deliver+0x270/0x270
[  136.471879]  __netif_receive_skb_core+0x103f/0x18b0
[  136.473186]  ? nf_ingress+0x390/0x390
[  136.474015]  ? addrconf_rs_timer+0x2c8/0x450
[  136.475098]  ? __run_timers+0x501/0x580
[  136.476443]  ? __do_softirq+0x1a1/0x4a8
[  136.477782]  ? irq_exit+0xe1/0xf0
[  136.478546]  ? smp_apic_timer_interrupt+0xf9/0x290
[  136.479730]  ? apic_timer_interrupt+0x7d/0x90
[  136.480519]  ? native_safe_halt+0x2/0x10
[  136.481567]  ? do_idle+0x1a6/0x220
[  136.482504]  ? cpu_startup_entry+0xdb/0xe0
[  136.483665]  ? dst_cow_metrics_generic+0x8b/0x170
[  136.484767]  ? stack_access_ok+0x35/0x80
[  136.486376]  ? deref_stack_reg+0xa1/0xe0
[  136.488719]  ? __read_once_size_nocheck.constprop.6+0x10/0x10
[  136.491745]  ? update_sd_lb_stats+0xc60/0xc60
[  136.493639]  ? __orc_find+0x6b/0xc0
[  136.495171]  ? unwind_next_frame+0x41a/0xb90
[  136.497018]  ? __save_stack_trace+0x5e/0x100
[  136.498879]  ? stack_access_ok+0x35/0x80
[  136.500290]  ? deref_stack_reg+0xa1/0xe0
[  136.501780]  ? __read_once_size_nocheck.constprop.6+0x10/0x10
[  136.504274]  ? get_stack_info+0x37/0x160
[  136.505993]  ? __orc_find+0x6b/0xc0
[  136.507528]  ? unwind_next_frame+0x41a/0xb90
[  136.509388]  ? __save_stack_trace+0x5e/0x100
[  136.511187]  ? stack_access_ok+0x35/0x80
[  136.512885]  ? deref_stack_reg+0xa1/0xe0
[  136.514567]  ? __read_once_size_nocheck.constprop.6+0x10/0x10
[  136.516451]  ? kernel_text_address+0xec/0x100
[  136.517300]  ? get_stack_info+0x37/0x160
[  136.519156]  ? __orc_find+0x6b/0xc0
[  136.521413]  ? secondary_startup_64+0xa4/0xb0
[  136.523411]  ? unwind_next_frame+0x475/0xb90
[  136.525282]  ? secondary_startup_64+0xa5/0xb0
[  136.526484]  ? start_kernel+0x57c/0x5bd
[  136.527826]  ? process_backlog+0x124/0x2a0
[  136.528589]  process_backlog+0x124/0x2a0
[  136.529963]  net_rx_action+0x4db/0xba0
[  136.531134]  ? napi_complete_done+0x320/0x320
[  136.531962]  ? __do_softirq+0x1a1/0x4a8
[  136.533003]  ? irq_exit+0xe1/0xf0
[  136.534452]  ? apic_timer_interrupt+0x7d/0x90
[  136.536361]  ? native_safe_halt+0x2/0x10
[  136.538094]  ? default_idle+0x82/0x250
[  136.539747]  ? do_idle+0x1a6/0x220
[  136.541226]  ? cpu_startup_entry+0xdb/0xe0
[  136.543027]  ? start_kernel+0x57c/0x5bd
[  136.544679]  ? rcu_segcblist_future_gp_needed+0x48/0x80
[  136.546670]  ? cpu_needs_another_gp+0x2c0/0x2d0
[  136.548312]  ? print_other_cpu_stall+0x770/0x770
[  136.550165]  ? rcu_note_context_switch+0x2d0/0x2d0
[  136.551093]  ? ip6_dst_destroy+0x22f/0x290
[  136.553524]  ? rcu_segcblist_extract_pend_cbs+0xb0/0xb0
[  136.555998]  ? debug_object_activate+0x22f/0x390
[  136.557989]  ? debug_object_free+0x270/0x270
[  136.559875]  ? timerqueue_add+0xd2/0x100
[  136.561588]  ? pvclock_clocksource_read+0x12c/0x220
[  136.563709]  ? pvclock_read_flags+0x50/0x50
[  136.565536]  ? run_rebalance_domains+0x279/0x460
[  136.567208]  ? calc_load_nohz_stop+0x1a0/0x1a0
[  136.569162]  ? pvclock_clocksource_read+0x12c/0x220
[  136.571273]  ? pvclock_read_flags+0x50/0x50
[  136.573010]  ? do_gettimeofday+0x100/0x100
[  136.574616]  ? _find_next_bit+0x30/0xb0
[  136.575641]  ? _find_next_bit+0x30/0xb0
[  136.576230]  ? timekeeping_max_deferment+0xa3/0x120
[  136.576952]  ? raise_softirq_irqoff+0x170/0x170
[  136.577625]  ? pvclock_clocksource_read+0x12c/0x220
[  136.578400]  __do_softirq+0x1a1/0x4a8
[  136.579638]  ? __irqentry_text_end+0x1fbd32/0x1fbd32
[  136.580819]  ? __napi_schedule_irqoff+0x120/0x120
[  136.582058]  ? flush_smp_call_function_queue+0x178/0x2b0
[  136.584146]  ? do_nothing+0x10/0x10
[  136.585775]  ? clockevents_program_event+0x100/0x130
[  136.587918]  ? hrtimer_interrupt+0x26a/0x2b0
[  136.588709]  irq_exit+0xe1/0xf0
[  136.589207]  smp_call_function_single_interrupt+0xc0/0x230
[  136.590059]  ? smp_call_function_interrupt+0x230/0x230
[  136.591863]  ? handle_irq+0x10f/0x1c0
[  136.592514]  ? rcu_eqs_enter_common.constprop.70+0x7d/0xb0
[  136.593431]  call_function_single_interrupt+0x7d/0x90
[  136.594285]  </IRQ>
[  136.594695] RIP: 0010:native_safe_halt+0x2/0x10
[  136.596290] RSP: 0018:ffffffffad207cf0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff04
[  136.598511] RAX: 0000000000000000 RBX: 0000000000000000 RCX: ffffffffac665fbc
[  136.599864] RDX: dffffc0000000000 RSI: 0000000000000000 RDI: ffffffffad218bc0
[  136.600959] RBP: 1ffffffff5a40f9f R08: fffffbfff5ac6e68 R09: fffffbfff5ac6e68
[  136.602082] R10: 0000000000000002 R11: fffffbfff5ac6e67 R12: ffffffffad638d50
[  136.603180] R13: 0000000000000000 R14: ffffffffad218bc0 R15: 0000000000000000
[  136.604282]  ? default_idle_call+0x1c/0x40
[  136.604937]  default_idle+0x82/0x250
[  136.606005]  ? __cpuidle_text_start+0x8/0x8
[  136.607417]  ? tsc_verify_tsc_adjust+0x176/0x1c0
[  136.608487]  ? wrmsrl.constprop.0+0x20/0x20
[  136.610129]  ? nohz_balance_exit_idle+0x50/0x50
[  136.612131]  do_idle+0x1a6/0x220
[  136.612942]  cpu_startup_entry+0xdb/0xe0
[  136.614070]  ? cpu_in_idle+0x20/0x20
[  136.615601]  ? transfer_pid+0x1f0/0x1f0
[  136.616356]  start_kernel+0x57c/0x5bd
[  136.617623]  ? mem_encrypt_init+0x6/0x6
[  136.618952]  ? x86_family+0x5/0x20
[  136.619583]  ? load_ucode_bsp+0x46/0x154
[  136.620255]  secondary_startup_64+0xa5/0xb0
[  136.620972] Code: 00 00 31 c0 e8 9f e7 0e eb f6 05 04 21 4a 00 01 66 0f ef c0 48 c7 84 24 70 01 00 00 00 00 00 00 c7 84 24 78 01 00 00 00 00 00 00 <0f> 29 84 24 60 01 00 00 74 20 49 8d 86 90 00 00 00 48 89 c7 48 
[  136.626091] RIP: tfw_http_sess_obtain+0xe3/0x690 [tempesta_fw] RSP: ffff88006ca06088
krizhanovsky commented 5 years ago

Just hit the test crash again on current master. I attach my kernel config for the issue reproducing. linux-4.14.32-config.txt

[  774.945263] tfw_test: TEST_RUN(http_sticky, sending_302_without_preparing)
[  774.947410] BUG: spinlock wrong owner on CPU#3, insmod/4351
[  774.949145] kasan: CONFIG_KASAN_INLINE enabled
[  774.949145] kasan: GPF could be caused by NULL-ptr deref or user memory access
[  774.949160] general protection fault: 0000 [#1] SMP KASAN
[  774.949162] Modules linked in: tfw_test(O+) tfw_fuzzer(O) tempesta_fw(O) tempesta_db(O) tempesta_tls(O) tempesta_lib(O) binfmt_misc bochs_drm ttm drm_kms_helper drm fb_sys_fops syscopyarea crct10dif_pclmul ppdev sysfillrect sg sysimgblt serio_raw parport_pc button parport pcspkr ip_tables x_tables ext4 crc16 mbcache jbd2 fscrypto sr_mod sd_mod cdrom ata_generic ata_piix psmouse libata scsi_mod e1000 i2c_piix4
[  774.949247] CPU: 3 PID: 4351 Comm: insmod Tainted: G           O    4.14.32-kdump+ #127
[  774.949249] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20180724_192412-buildhw-07.phx2.fedoraproject.org-1.fc29 04/01/2014
[  774.949250] task: ffff880116c59080 task.stack: ffff880101b00000
[  774.949262] RIP: 0010:spin_dump+0xe3/0x280
[  774.949263] RSP: 0018:ffff880101b07298 EFLAGS: 00010202
[  774.949265] RAX: dffffc0000000000 RBX: ffffffffc1059bf8 RCX: 0000000000000006
[  774.949266] RDX: 0000000022d8b292 RSI: 0000000000000082 RDI: 0000000116c59490
[  774.949267] RBP: 0000000116c59080 R08: ffff88011b400000 R09: 0000000000000003
[  774.949268] R10: 0000000000016033 R11: 1ffffffff5a00a77 R12: ffff880116c59080
[  774.949268] R13: ffff880101b07340 R14: dffffc0000000000 R15: ffff880101b07340
[  774.949270] FS:  00007fda21ff2700(0000) GS:ffff88011b580000(0000) knlGS:0000000000000000
[  774.949271] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  774.949274] CR2: 000055eb87d268d8 CR3: 000000011a68a004 CR4: 00000000003606e0
[  774.949275] Call Trace:
[  774.949302]  ? tfw_http_msg_free+0x1c6/0x2d0 [tfw_test]
[  774.949305]  do_raw_spin_unlock+0x173/0x240
[  774.949312]  _raw_spin_unlock_bh+0x5/0x20
[  774.949329]  tfw_http_resp_fwd+0x588/0xc50 [tfw_test]
[  774.949346]  ? tfw_http_conn_resend+0x1af0/0x1af0 [tfw_test]
[  774.949348]  ? _raw_spin_unlock+0x5/0x10
[  774.949363]  test_suite__http_sticky+0x486/0x47a0 [tfw_test]
[  774.949378]  ? test_helper_sticky_stop+0x30/0x30 [tfw_test]
[  774.949381]  ? soft_cursor+0x5bd/0xc40
[  774.949382]  ? bit_clear+0x540/0x540
[  774.949386]  ? get_color+0x2a/0x3a0
[  774.949403]  ? test_suite__addr+0xf0/0xf0 [tfw_test]
[  774.949405]  ? printk+0x97/0xbe
[  774.949406]  ? show_regs_print_info+0x53/0x53
[  774.949423]  ? tfw_http_msg_free+0x1c6/0x2d0 [tfw_test]
[  774.949424]  ? 0xffffffffc1480000
[  774.949440]  test_run_all+0x230/0x390 [tfw_test]
[  774.949442]  ? 0xffffffffc1480000
[  774.949456]  tfw_test_init+0x14/0x1000 [tfw_test]
[  774.949459]  do_one_initcall+0x90/0x210
[  774.949460]  ? initcall_blacklisted+0x150/0x150
[  774.949465]  ? kasan_kmalloc+0xa0/0xd0
[  774.949467]  ? kmem_cache_alloc_trace+0xd1/0x570