Closed kingwelx closed 5 years ago
This can be reproduced by the same code provided in the report. It probably only happens when calling flush_job, when there are very few jobs submitted. I tried submitting a bunch of jobs, but I cannot observe the crash – in this case, jobs will be completed by submit_job itself. So I think the key things here are:
Occasionally, I also observed crash at the same place even length > 16. However, I am not so sure about it, because I did a lot of tests, confused sometime?? Anyway, it has be to the same root cause, I guess.
Tested on v0.51 and .52, same results.
Thanks for reporting the problem. We'll have a look into it.
Sorry for the mistake in the sample code, it should be like this to call flush_job for CAMC twice:
static int do_test2 (MB_MGR mb_mgr) { uint32_t size; JOB_AES_HMAC job; static uint128_t IV = {0,0}; DECLARE_ALIGNED(uint32_t expkey[415], 16); uint32_t skey1[4], skey2[4]; static uint32_t digest[3]; uint128_t keys[15]; static uint8_t buf[4096+20]; for (size = 1; size < 3; size += 1) { job = IMB_GET_NEXT_JOB(mb_mgr); job->msg_len_to_cipher_in_bytes = size; job->msg_len_to_hash_in_bytes = size; job->hash_start_src_offset_in_bytes = 0; job->cipher_start_src_offset_in_bytes = 0; job->auth_tag_output = (uint8_t) digest; job->auth_tag_output_len_in_bytes = 12; job->u.CMAC._key_expanded = expkey; job->u.CMAC._skey1 = skey1; job->u.CMAC._skey2 = skey2; job->aes_enc_key_expanded = job->aes_dec_key_expanded = (uint32_t) keys; job->src = buf; job->dst = buf; job->iv = (uint8_t ) &IV; job->iv_len_in_bytes = 16; job->cipher_mode = NULL_CIPHER; job->hash_alg = AES_CMAC; job->aes_key_len_in_bytes = 16; job->cipher_direction = DECRYPT; job->chain_order = HASH_CIPHER; job = IMB_SUBMIT_JOB(mb_mgr); while (job) { job = IMB_GET_COMPLETED_JOB(mb_mgr); } / end while (job) / while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { } } return 0; }
Hi Kingwel, could you verify that the patch fixes the problem please?
Hi Kingwel,
Thanks for reporting this issue. It seems that the AES-CMAC flush function was overwriting the job lengths when the job size was less than 16 bytes. We will update the test application to cover this case soon.
Regards, Marcel
Test passed, no crash observed.
Many thanks!
Test code:
static int do_test2 (MB_MGR mb_mgr) { uint32_t size; JOB_AES_HMAC job; static uint128_t IV = {0,0};
DECLARE_ALIGNED(uint32_t expkey[4*15], 16);
uint32_t skey1[4], skey2[4];
static uint32_t digest[3];
uint128_t keys[15];
static uint8_t buf[4096+20];
for (size = 1; size < 3; size += 1) {
job = IMB_GET_NEXT_JOB(mb_mgr);
job->msg_len_to_cipher_in_bytes = size;
job->msg_len_to_hash_in_bytes = size;
job->hash_start_src_offset_in_bytes = 0;
job->cipher_start_src_offset_in_bytes = 0;
job->auth_tag_output = (uint8_t*) digest;
job->auth_tag_output_len_in_bytes = 12;
job->u.CMAC._key_expanded = expkey;
job->u.CMAC._skey1 = skey1;
job->u.CMAC._skey2 = skey2;
job->aes_enc_key_expanded =
job->aes_dec_key_expanded = (uint32_t*) keys;
job->src = buf;
job->dst = buf;
job->iv = (uint8_t *) &IV;
job->iv_len_in_bytes = 16;
job->cipher_mode = NULL_CIPHER;
job->hash_alg = AES_CMAC;
job->aes_key_len_in_bytes = 16;
job->cipher_direction = DECRYPT;
job->chain_order = HASH_CIPHER;
job = IMB_SUBMIT_JOB(mb_mgr);
while (job) {
printf ("Sstatus = %d\n", job->status);
job = IMB_GET_COMPLETED_JOB(mb_mgr);
} /* end while (job) */
while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) {
printf ("status = %d\n", job->status);
}
}
return 0;
}
Thanks! closing the issue
Please check the call stack.As shown below, t looks like the rax is 0, when addressing job_rax + _auth_tag_output_len_in_bytes.
(gdb)
0 0x00007fffb3086c04 in ..@208._copy_complete_digest () from /work/vpp/build-root/install-vpp-native/vpp/lib/vpp_plugins/ppfu_plugin.so
1 0x00000000006eca00 in ?? ()
2 0x00000000006ecbe0 in ?? ()
3 0x00000000006f2b80 in ?? ()
4 0x00000000006f2d40 in ?? ()
5 0x00000000006f29c0 in ?? ()
6 0x00007fffb899d610 in ?? ()
7 0x0000000000000001 in ?? ()
8 0x00007ffff6cf26ef in clib_mem_size (p=0x7fffb30d64cf <flush_job_avx2+143>) at /work/vpp/src/vppinfra/mem.h:250
9 _vec_resize_inline (data_align=4, header_bytes=0, data_bytes=112, length_increment=1, v=0x7fffb30d64d7 <flush_job_avx2+151>) at /work/vpp/src/vppinfra/vec.h:139
10 vlib_put_next_frame (vm=0x0, r=0x7ffbb35393e1, next_index=, n_vectors_left=) at /work/vpp/src/vlib/main.c:504
11 0x00000000006eca00 in ?? ()
12 0x0000000000000000 in ?? ()
(gdb) disass Dump of assembler code for function ..@208._copy_complete_digest: 0x00007fffb3086bed <+0>: mov 0x168(%rdi,%rbp,8),%rax 0x00007fffb3086bf5 <+8>: mov %rbp,%r13 0x00007fffb3086bf8 <+11>: shl $0x4,%r13 0x00007fffb3086bfc <+15>: lea 0xc0(%rdi,%r13,1),%r12 => 0x00007fffb3086c04 <+23>: mov 0x60(%rax),%r13 0x00007fffb3086c08 <+27>: mov 0x58(%rax),%r14 0x00007fffb3086c0c <+31>: cmp $0x10,%r13 0x00007fffb3086c10 <+35>: jne 0x7fffb3086c1f ..@211.lt16 0x00007fffb3086c12 <+37>: vmovdqa (%r12),%xmm0 0x00007fffb3086c18 <+43>: vmovdqu %xmm0,(%r14) 0x00007fffb3086c1d <+48>: jmp 0x7fffb3086c87 ..@211.end End of assembler dump.
(gdb) info reg rax 0x0 0 rbx 0xf0123456 4027724886 rcx 0x0 0 rdx 0xffffffffff906d4b -7312053 rsi 0x0 0 rdi 0x6fc240 7324224 rbp 0x0 0x0 rsp 0x7fffb63ff340 0x7fffb63ff340 r8 0x7 7 r9 0x10081008a000000 72199433353166848 r10 0x6fc470 7324784 r11 0x1 1 r12 0x6fc300 7324416 r13 0x0 0 r14 0x7ffbb35393e1 140719022117857 r15 0x70 112 rip 0x7fffb3086c04 0x7fffb3086c04 ..@208._copy_complete_digest+23 eflags 0x13246 [ PF ZF IF #12 #13 RF ] cs 0x33 51 ss 0x2b 43 ds 0x0 0 es 0x0 0 fs 0x0 0 gs 0x0 0
source code: %%_copy_complete_digest: ; Job complete, copy digest to AT output mov job_rax, [state + _aes_cmac_job_in_lane + idx*8]
Test code shown below:
static int do_test2 (MB_MGR mb_mgr) { uint32_t size; JOB_AES_HMAC job; static uint128_t IV = {0,0}; DECLARE_ALIGNED(uint32_t expkey[415], 16); uint32_t skey1[4], skey2[4]; static uint32_t digest[3]; uint128_t keys[15]; static uint8_t buf[4096+20]; for (size = 1; size < 2; size += 1) { job = IMB_GET_NEXT_JOB(mb_mgr); job->msg_len_to_cipher_in_bytes = size; job->msg_len_to_hash_in_bytes = size; job->hash_start_src_offset_in_bytes = 0; job->cipher_start_src_offset_in_bytes = 0; job->auth_tag_output = (uint8_t) digest; job->auth_tag_output_len_in_bytes = 12; job->u.CMAC._key_expanded = expkey; job->u.CMAC._skey1 = skey1; job->u.CMAC._skey2 = skey2; job->aes_enc_key_expanded = job->aes_dec_key_expanded = (uint32_t) keys; job->src = buf; job->dst = buf; job->iv = (uint8_t ) &IV; job->iv_len_in_bytes = 16; job->cipher_mode = NULL_CIPHER; job->hash_alg = AES_CMAC; job->aes_key_len_in_bytes = 16; job->cipher_direction = DECRYPT; job->chain_order = HASH_CIPHER; job = IMB_SUBMIT_JOB(mb_mgr); while (job) { job = IMB_GET_COMPLETED_JOB(mb_mgr); } / end while (job) / } while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { } return 0; }