AES-CMAC crashed in flush_job when msg_len_to_hash_in_bytes is <= 16bytes

[kingwelx]

Please check the call stack.As shown below, t looks like the rax is 0, when addressing job_rax + _auth_tag_output_len_in_bytes.

(gdb)

0 0x00007fffb3086c04 in ..@208._copy_complete_digest () from /work/vpp/build-root/install-vpp-native/vpp/lib/vpp_plugins/ppfu_plugin.so

1 0x00000000006eca00 in ?? ()

2 0x00000000006ecbe0 in ?? ()

3 0x00000000006f2b80 in ?? ()

4 0x00000000006f2d40 in ?? ()

5 0x00000000006f29c0 in ?? ()

6 0x00007fffb899d610 in ?? ()

7 0x0000000000000001 in ?? ()

8 0x00007ffff6cf26ef in clib_mem_size (p=0x7fffb30d64cf <flush_job_avx2+143>) at /work/vpp/src/vppinfra/mem.h:250

9 _vec_resize_inline (data_align=4, header_bytes=0, data_bytes=112, length_increment=1, v=0x7fffb30d64d7 <flush_job_avx2+151>) at /work/vpp/src/vppinfra/vec.h:139

10 vlib_put_next_frame (vm=0x0, r=0x7ffbb35393e1, next_index=, n_vectors_left=) at /work/vpp/src/vlib/main.c:504

11 0x00000000006eca00 in ?? ()

12 0x0000000000000000 in ?? ()

(gdb) disass Dump of assembler code for function ..@208._copy_complete_digest: 0x00007fffb3086bed <+0>: mov 0x168(%rdi,%rbp,8),%rax 0x00007fffb3086bf5 <+8>: mov %rbp,%r13 0x00007fffb3086bf8 <+11>: shl $0x4,%r13 0x00007fffb3086bfc <+15>: lea 0xc0(%rdi,%r13,1),%r12 => 0x00007fffb3086c04 <+23>: mov 0x60(%rax),%r13 0x00007fffb3086c08 <+27>: mov 0x58(%rax),%r14 0x00007fffb3086c0c <+31>: cmp $0x10,%r13 0x00007fffb3086c10 <+35>: jne 0x7fffb3086c1f ..@211.lt16 0x00007fffb3086c12 <+37>: vmovdqa (%r12),%xmm0 0x00007fffb3086c18 <+43>: vmovdqu %xmm0,(%r14) 0x00007fffb3086c1d <+48>: jmp 0x7fffb3086c87 ..@211.end End of assembler dump.

(gdb) info reg rax 0x0 0 rbx 0xf0123456 4027724886 rcx 0x0 0 rdx 0xffffffffff906d4b -7312053 rsi 0x0 0 rdi 0x6fc240 7324224 rbp 0x0 0x0 rsp 0x7fffb63ff340 0x7fffb63ff340 r8 0x7 7 r9 0x10081008a000000 72199433353166848 r10 0x6fc470 7324784 r11 0x1 1 r12 0x6fc300 7324416 r13 0x0 0 r14 0x7ffbb35393e1 140719022117857 r15 0x70 112 rip 0x7fffb3086c04 0x7fffb3086c04 ..@208._copy_complete_digest+23 eflags 0x13246 [ PF ZF IF #12 #13 RF ] cs 0x33 51 ss 0x2b 43 ds 0x0 0 es 0x0 0 fs 0x0 0 gs 0x0 0

---

source code: %%_copy_complete_digest: ; Job complete, copy digest to AT output mov job_rax, [state + _aes_cmac_job_in_lane + idx*8]

    mov     tmp4, idx
    shl     tmp4, 4
    lea     tmp3, [state + _aes_cmac_args_IV + tmp4]
    mov     tmp4, [job_rax + _auth_tag_output_len_in_bytes]
    mov     tmp2, [job_rax + _auth_tag_output]

    cmp     tmp4, 16
    jne     %%_ne_16_copy

    ;; 16 byte AT copy
    vmovdqa xmm0, [tmp3]
    vmovdqu [tmp2], xmm0
    jmp     %%_update_lanes

Test code shown below:

static int do_test2 (MB_MGR mb_mgr) { uint32_t size; JOB_AES_HMAC job; static uint128_t IV = {0,0}; DECLARE_ALIGNED(uint32_t expkey[415], 16); uint32_t skey1[4], skey2[4]; static uint32_t digest[3]; uint128_t keys[15]; static uint8_t buf[4096+20]; for (size = 1; size < 2; size += 1) { job = IMB_GET_NEXT_JOB(mb_mgr); job->msg_len_to_cipher_in_bytes = size; job->msg_len_to_hash_in_bytes = size; job->hash_start_src_offset_in_bytes = 0; job->cipher_start_src_offset_in_bytes = 0; job->auth_tag_output = (uint8_t) digest; job->auth_tag_output_len_in_bytes = 12; job->u.CMAC._key_expanded = expkey; job->u.CMAC._skey1 = skey1; job->u.CMAC._skey2 = skey2; job->aes_enc_key_expanded = job->aes_dec_key_expanded = (uint32_t) keys; job->src = buf; job->dst = buf; job->iv = (uint8_t ) &IV; job->iv_len_in_bytes = 16; job->cipher_mode = NULL_CIPHER; job->hash_alg = AES_CMAC; job->aes_key_len_in_bytes = 16; job->cipher_direction = DECRYPT; job->chain_order = HASH_CIPHER; job = IMB_SUBMIT_JOB(mb_mgr); while (job) { job = IMB_GET_COMPLETED_JOB(mb_mgr); } / end while (job) / } while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { } return 0; }

[kingwelx]

This can be reproduced by the same code provided in the report. It probably only happens when calling flush_job, when there are very few jobs submitted. I tried submitting a bunch of jobs, but I cannot observe the crash – in this case, jobs will be completed by submit_job itself. So I think the key things here are:

1. Length <= 16
2. Only one job in lane to be completed by flush_job

Occasionally, I also observed crash at the same place even length > 16. However, I am not so sure about it, because I did a lot of tests, confused sometime?? Anyway, it has be to the same root cause, I guess.

[kingwelx]

Tested on v0.51 and .52, same results.

[tkanteck]

Thanks for reporting the problem. We'll have a look into it.

[kingwelx]

Sorry for the mistake in the sample code, it should be like this to call flush_job for CAMC twice:

static int do_test2 (MB_MGR mb_mgr) { uint32_t size; JOB_AES_HMAC job; static uint128_t IV = {0,0}; DECLARE_ALIGNED(uint32_t expkey[415], 16); uint32_t skey1[4], skey2[4]; static uint32_t digest[3]; uint128_t keys[15]; static uint8_t buf[4096+20]; for (size = 1; size < 3; size += 1) { job = IMB_GET_NEXT_JOB(mb_mgr); job->msg_len_to_cipher_in_bytes = size; job->msg_len_to_hash_in_bytes = size; job->hash_start_src_offset_in_bytes = 0; job->cipher_start_src_offset_in_bytes = 0; job->auth_tag_output = (uint8_t) digest; job->auth_tag_output_len_in_bytes = 12; job->u.CMAC._key_expanded = expkey; job->u.CMAC._skey1 = skey1; job->u.CMAC._skey2 = skey2; job->aes_enc_key_expanded = job->aes_dec_key_expanded = (uint32_t) keys; job->src = buf; job->dst = buf; job->iv = (uint8_t ) &IV; job->iv_len_in_bytes = 16; job->cipher_mode = NULL_CIPHER; job->hash_alg = AES_CMAC; job->aes_key_len_in_bytes = 16; job->cipher_direction = DECRYPT; job->chain_order = HASH_CIPHER; job = IMB_SUBMIT_JOB(mb_mgr); while (job) { job = IMB_GET_COMPLETED_JOB(mb_mgr); } / end while (job) / while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) { } } return 0; }

[tkanteck]

Hi Kingwel, could you verify that the patch fixes the problem please?

[mdcornu]

Hi Kingwel,

Thanks for reporting this issue. It seems that the AES-CMAC flush function was overwriting the job lengths when the job size was less than 16 bytes. We will update the test application to cover this case soon.

Regards, Marcel

[kingwelx]

Test passed, no crash observed.

Many thanks!

Test code:

static int do_test2 (MB_MGR mb_mgr) { uint32_t size; JOB_AES_HMAC job; static uint128_t IV = {0,0};

    DECLARE_ALIGNED(uint32_t expkey[4*15], 16);

    uint32_t skey1[4], skey2[4];

    static uint32_t digest[3];
    uint128_t keys[15];
    static uint8_t buf[4096+20];
for (size = 1; size < 3; size += 1) {

        job = IMB_GET_NEXT_JOB(mb_mgr);

        job->msg_len_to_cipher_in_bytes = size;
        job->msg_len_to_hash_in_bytes = size;
        job->hash_start_src_offset_in_bytes = 0;
        job->cipher_start_src_offset_in_bytes = 0;

        job->auth_tag_output = (uint8_t*) digest;
        job->auth_tag_output_len_in_bytes = 12;

        job->u.CMAC._key_expanded = expkey;
        job->u.CMAC._skey1 = skey1;
        job->u.CMAC._skey2 = skey2;

        job->aes_enc_key_expanded =
                job->aes_dec_key_expanded = (uint32_t*) keys;
        job->src = buf;
        job->dst = buf;
        job->iv = (uint8_t *) &IV;
        job->iv_len_in_bytes = 16;

        job->cipher_mode = NULL_CIPHER;
        job->hash_alg = AES_CMAC;

            job->aes_key_len_in_bytes = 16;
    job->cipher_direction = DECRYPT;
    job->chain_order = HASH_CIPHER;

        job = IMB_SUBMIT_JOB(mb_mgr);
        while (job) {
        printf ("Sstatus = %d\n", job->status);
                job = IMB_GET_COMPLETED_JOB(mb_mgr);
        } /* end while (job) */

        while ((job = IMB_FLUSH_JOB(mb_mgr)) != NULL) {
        printf ("status = %d\n", job->status);
        }
}

    return 0;

}

[tkanteck]

Thanks! closing the issue