search

tempesta-tech / tempesta

All-in-one solution for high performance web content delivery and advanced protection against DDoS and web attacks
https://tempesta-tech.com/
GNU General Public License v2.0
613 stars 103 forks source link

BUG: kernel NULL pointer dereference: tfw_hpack_set_entry #2187

Closed EvgeniiMekhanik closed 1 week ago

EvgeniiMekhanik commented 1 month ago 
tempesta login: [17411.944170] BUG: kernel NULL pointer dereference, address: 0000000000000001
[17411.953441] #PF: supervisor read access in kernel mode
[17411.955857] #PF: error_code(0x0000) - not-present page
[17411.958242] PGD 0 P4D 0 
[17411.960303] Oops: 0000 [#1] SMP PTI
[17411.962576] CPU: 2 PID: 0 Comm: swapper/2 Tainted: G        W  OE     5.10.35+ #4
[17411.965271] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.15.0-1 04/01/2014
[17411.968027] RIP: 0010:__memcpy_fast+0x13d/0x160 [tempesta_lib]
[17411.970340] Code: 83 c0 08 48 83 c1 08 48 89 70 f8 f6 c2 04 0f 84 25 ff ff ff 8b 31 48 83 c0 04 48 83 c1 04 89 70 fc f6 c2 02 0f 84 18 ff ff ff <0f> b7 31 48 83 c0 02 48 83 c1 02 66 89 70 fe 83 e2 01 74 05 0f b6
[17411.976014] RSP: 0018:ffffba9e401b4738 EFLAGS: 00010202
[17411.978254] RAX: ffff9bab695e205d RBX: ffff9babf4897600 RCX: 0000000000000001
[17411.980907] RDX: 0000000000000003 RSI: 0000000000000001 RDI: ffff9bab695e205d
[17411.983379] RBP: ffffba9e401b4740 R08: 0000000000000001 R09: ffff9bab68c7e900
[17411.985934] R10: ffff9babcf4ab020 R11: ffff9babf4897010 R12: ffff9babcf4ab040
[17411.988454] R13: ffff9babcf4ab040 R14: ffff9bab695e205d R15: ffff9babf4897020
[17411.990806] FS:  0000000000000000(0000) GS:ffff9baeaf680000(0000) knlGS:0000000000000000
[17411.993389] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[17411.995511] CR2: 0000000000000001 CR3: 00000001239e2006 CR4: 0000000000770ee0
[17411.997873] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[17412.000202] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[17412.002455] PKRU: 55555554
[17412.004133] Call Trace:
[17412.005790]  <IRQ>
[17412.007309]  ? memcpy_fast+0xe/0x10 [tempesta_lib]
[17412.009251]  tfw_hpack_decode+0x1532/0x2300 [tempesta_fw]
[17412.011150]  tfw_h2_parse_req+0x1a6/0x320 [tempesta_fw]
[17412.013124]  ss_skb_process+0xf5/0x140 [tempesta_fw]
[17412.014877]  ? h2_set_hdr_x_method_override+0x20/0x20 [tempesta_fw]
[17412.016951]  tfw_http_req_process+0x97/0xa40 [tempesta_fw]
[17412.018716]  ? __tfw_pool_new+0x24/0x70 [tempesta_fw]
[17412.020590]  ? bzero_fast+0xe/0x10 [tempesta_lib]
[17412.022263]  ? __tfw_http_msg_alloc+0x25b/0x340 [tempesta_fw]
[17412.024146]  ? bzero_fast+0xe/0x10 [tempesta_lib]
[17412.025815]  ? tfw_http_init_parser_req+0x25/0x60 [tempesta_fw]
[17412.027617]  ? __tfw_pool_new+0x24/0x70 [tempesta_fw]
[17412.029352]  tfw_http_msg_process_generic+0x188/0x6e0 [tempesta_fw]
[17412.031138]  ? ss_skb_process+0xf5/0x140 [tempesta_fw]
[17412.032863]  ? tfw_h2_send_rst_stream+0xb0/0xb0 [tempesta_fw]
[17412.034509]  tfw_h2_frame_process+0x42b/0x6b0 [tempesta_fw]
[17412.036235]  tfw_http_msg_process+0x48/0x60 [tempesta_fw]
[17412.037848]  tfw_connection_recv+0xbb/0x140 [tempesta_fw]
[17412.039424]  tfw_tls_connection_recv+0x332/0x450 [tempesta_fw]
[17412.041129]  ss_tcp_process_data+0x20a/0x4b0 [tempesta_fw]
[17412.042625]  ss_tcp_data_ready+0x57/0x140 [tempesta_fw]
[17412.044209]  tcp_data_ready+0x2b/0xd0
[17412.045484]  tcp_data_queue+0x805/0xe50
[17412.046696]  tcp_rcv_established+0x254/0x910
[17412.048078]  tcp_v4_do_rcv+0x140/0x200
[17412.049325]  tcp_v4_rcv+0xcd0/0xe20
[17412.050452]  ip_protocol_deliver_rcu+0x44/0x230
[17412.051815]  ip_local_deliver_finish+0x48/0x60
[17412.053173]  ip_local_deliver+0x70/0x110
[17412.054346]  ? ip_rcv_finish_core.constprop.0+0x172/0x470
[17412.055813]  ip_rcv_finish+0x87/0xa0
[17412.057027]  ip_rcv+0xce/0xe0
[17412.058033]  ? ip_rcv_finish_core.constprop.0+0x470/0x470
[17412.059408]  __netif_receive_skb_one_core+0x86/0xa0
[17412.060764]  __netif_receive_skb+0x18/0x60
[17412.061920]  process_backlog+0x9e/0x170
[17412.063033]  net_rx_action+0x13b/0x430
[17412.064198]  __do_softirq+0xe3/0x340
[17412.065312]  asm_call_irq_on_stack+0x12/0x20
[17412.066512]  </IRQ>
[17412.067400]  do_softirq_own_stack+0x3d/0x50
[17412.068613]  irq_exit_rcu+0xa2/0xe0
[17412.069713]  sysvec_call_function_single+0x3d/0x90
[17412.070942]  asm_sysvec_call_function_single+0x12/0x20
[17412.072285] RIP: 0010:native_safe_halt+0xe/0x10
[17412.073502] Code: 39 ff ff ff 4c 89 ee 48 c7 c7 a0 ba 85 b0 e8 89 64 91 ff e9 01 ff ff ff cc cc cc cc e9 07 00 00 00 0f 00 2d 96 55 47 00 fb f4 <c3> 90 e9 07 00 00 00 0f 00 2d 86 55 47 00 f4 c3 cc cc 0f 1f 44 00
[17412.077365] RSP: 0018:ffffba9e40093e88 EFLAGS: 00000212
[17412.078659] RAX: ffffffffaf996750 RBX: 0000000000000002 RCX: ffff9baeaf6acdc0
[17412.080377] RDX: 000000000081d2ae RSI: 0000000000000083 RDI: 0000000000000002
[17412.081996] RBP: ffffba9e40093e90 R08: ffffffffb0293970 R09: 0000000000000000
[17412.083672] R10: 00000fd605dd8019 R11: 0000000000000000 R12: ffff9bab6034dc40
[17412.085349] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
[17412.086935]  ? __sched_text_end+0x4/0x4
[17412.088149]  ? default_idle+0xe/0x20
[17412.089320]  arch_cpu_idle+0x15/0x20
[17412.090436]  default_idle_call+0x3d/0xc0
[17412.091629]  do_idle+0x215/0x2a0
[17412.092749]  cpu_startup_entry+0x20/0x30
[17412.093883]  start_secondary+0x145/0x1b0
[17412.095017]  secondary_startup_64_no_verify+0xc2/0xcb

e42225f5

/home/kingluo/tempesta/lib/str_simd.S:127
.L2cpy:
        movzwl  (%rcx), %esi
 9ed:   0f b7 31                movzwl (%rcx),%esi ;<--------- %rcx=input_%rsi=s->data=0x01
...
/home/kingluo/tempesta/fw/hpack.c:741
                memcpy_fast(data, s->data, s->len);
   1b5e9:       e8 00 00 00 00          call   1b5ee <tfw_hpack_decode+0x156e>
                        1b5ea: R_X86_64_PLT32   __asan_load8_noabort-0x4
   1b5ee:       49 8b 56 e8             mov    -0x18(%r14),%rdx
   1b5f2:       49 8b 76 e0             mov    -0x20(%r14),%rsi  ; <----------- s->data
   1b5f6:       4c 89 e7                mov    %r12,%rdi
   1b5f9:       e8 00 00 00 00          call   1b5fe <tfw_hpack_decode+0x157e>
                        1b5fa: R_X86_64_PLT32   memcpy_fast-0x4
EvgeniiMekhanik commented 1 month ago 
listen 192.168.122.100:443 proto=h2,https;
listen 192.168.122.100:80 proto=http;

access_log on;
client_tbl_size 134217728;

block_action attack reply;
block_action error reply;

tls_certificate /home/tempesta/certs/tempesta.kinescope.io.pem;
tls_certificate_key /home/tempesta/certs/tempesta.kinescope.io.key;

srv_group main {
    server 192.168.122.1:443;
}

srv_group http {
    server 192.168.122.1:80;
}

vhost main {
        proxy_pass main;
}

vhost http {
    proxy_pass http;
}

http_chain {
    mark == 1 -> http;
    -> main;
}
EvgeniiMekhanik commented 1 month ago

duplicated with https://github.com/tempesta-tech/tempesta/issues/2206.

[12622.176271] BUG: kernel NULL pointer dereference, address: 0000000000000008
[12622.183542] #PF: supervisor read access in kernel mode
[12622.185797] #PF: error_code(0x0000) - not-present page
[12622.188109] PGD 0 P4D 0
[12622.190019] Oops: 0000 [#1] SMP PTI
[12622.191983] CPU: 15 PID: 0 Comm: swapper/15 Tainted: G           OE     5.10.35+ #4
[12622.194615] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.15.0-1 04/01/2014
[12622.197185] RIP: 0010:tfw_hpack_decode+0xcbf/0x2300 [tempesta_fw]
[12622.199473] Code: 4c 03 65 90 eb 1d 83 c3 01 49 83 c4 70 41 83 ee 01 39 5d a0 0f 84 6c 0a 00 00 45 39 ef 0f 83 0d 0a 00 00 49 8b 34 24 44 89 c7 <2b> 7e 08 41 01 fd 41 f6 44 24 1c 01 74 ce 49 8b 79 50 4c 89 4d 88
[12622.204951] RSP: 0018:ffffa4cc40458718 EFLAGS: 00010282
[12622.207185] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 000000000000009d
[12622.209526] RDX: 0000000000000f63 RSI: 0000000000000000 RDI: 00000000ffffffe0
[12622.212066] RBP: ffffa4cc404587b8 R08: 00000000ffffffe0 R09: ffff99046ed208f8
[12622.214523] R10: 0000000000000000 R11: ffff9903eb78610b R12: ffff9903ebd62010
[12622.216944] R13: 0000000000000fd7 R14: 0000000000000021 R15: 0000000000000f63
[12622.219317] FS:  0000000000000000(0000) GS:ffff99072f9c0000(0000) knlGS:0000000000000000
[12622.221814] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[12622.223963] CR2: 0000000000000008 CR3: 00000001b1112002 CR4: 0000000000770ee0
[12622.226304] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[12622.228598] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[12622.230889] PKRU: 55555554
[12622.232572] Call Trace:
[12622.234222]  <IRQ>
[12622.235749]  tfw_h2_parse_req+0x1a6/0x320 [tempesta_fw]
[12622.237680]  ss_skb_process+0xf5/0x140 [tempesta_fw]
[12622.239553]  ? h2_set_hdr_x_method_override+0x20/0x20 [tempesta_fw]
[12622.241582]  ? h2_set_hdr_x_method_override+0x20/0x20 [tempesta_fw]
[12622.243604]  tfw_http_req_process+0x9b/0xa60 [tempesta_fw]
[12622.245370]  ? tfw_pool_alloc_pages+0x53/0x60 [tempesta_fw]
[12622.247286]  ? __get_free_pages+0x11/0x40
[12622.248974]  ? tfw_pool_alloc_pages+0x53/0x60 [tempesta_fw]
[12622.249300] [tempesta fw] 172.115.248.175 "player-metrics.kinescope.io.vhost_main" "POST /v1/batch/gzip?referrer=https%3A%2F%2Frevitonica.ru%2F HTTP/2.0" 200 0 "https://kinescope.io/" "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
[12622.250871]  ? __tfw_pool_new+0x24/0x70 [tempesta_fw]
[12622.259429]  tfw_http_msg_process_generic+0x1c3/0x900 [tempesta_fw]
[12622.261407]  ? __put_page+0x4f/0x90
[12622.262967]  ? memcpy_fast+0xe/0x10 [tempesta_lib]
[12622.264648]  tfw_h2_frame_process+0x42b/0x6b0 [tempesta_fw]
[12622.266039] [tempesta fw] 188.170.83.37 "player-metrics.kinescope.io.vhost_main" 
0x1544f is in tfw_hpack_decode (/home/tempesta/tempesta/fw/hpack.c:832).
827                    "maximum allowed decreased size: %u\n",  __func__,
828                    curr, early, count, window);
829 
830             cp = entries + early;
831             do {
832                 size -= HPACK_ENTRY_OVERHEAD + cp->hdr->len;
833                 T_DBG3("%s: dropped index: %u\n", __func__,
834                        early);
835                 if (cp->last)
836                     tfw_pool_clean_single(tbl->h_pool,
krizhanovsky commented 3 weeks ago

Fixed by #2208

EvgeniiMekhanik commented 1 week ago

The problem was in memory corruption in hpack, which corrupt memory in random place.