isal_inflate with crc_flag==ISAL_DEFLATE eats extra bytes.

[kirbyzhou]

isal_inflate with crc_flag==ISAL_DEFLATE eats extra bytes when there are extra bytes at tail. It always want to pick 64 bit at once, so if we provide extra bytes to it, it will eat it.

The demo code provides 8 extra bytes in isal_inflate, it eats 6.

#define IN_DATA_SIZE ((ssize_t)(128))
int main()
{
    uint8_t indata[IN_DATA_SIZE];
    uint8_t out1[IN_DATA_SIZE*2];
    ssize_t out1_len;
    uint8_t out2[IN_DATA_SIZE];
    ssize_t out2_len;
    srand(0);
    for (int i = 0; i < IN_DATA_SIZE; ++i) {
        indata[i] = rand()%64;
    }

    int level = -1;
    int nowrap = 1;
    fprintf(stderr, "level=%d nowrap=%d\n", level, nowrap);
    {
        z_stream *strm = calloc(1, sizeof(z_stream));
        deflateInit2(strm, level, Z_DEFLATED, nowrap ? -MAX_WBITS : MAX_WBITS, 8, Z_DEFAULT_STRATEGY);

        strm->avail_in = 0;

        strm->next_in = indata;
        strm->avail_in = IN_DATA_SIZE;
        strm->next_out = out1;
        strm->avail_out = sizeof(out1);
        int r = deflate(strm, Z_FINISH);

        assert(r == Z_STREAM_END);
        out1_len = sizeof(out1) - strm->avail_out;
        fprintf(stderr, "%zd -> %zd r=%d\n", IN_DATA_SIZE, out1_len, r);
    }
    {
        struct inflate_state *strm = calloc(1, sizeof(struct inflate_state));
        isal_inflate_init(strm);
        strm->crc_flag = nowrap ? ISAL_DEFLATE : ISAL_ZLIB;
        strm->next_in = out1;
        strm->avail_in = out1_len+8;
        strm->next_out = out2;
        strm->avail_out = sizeof(out2);
        fprintf(stderr, "pre isal_inflate avail_in=%d\n", strm->avail_in);
        int r = isal_inflate(strm);
        fprintf(stderr, "post isal_inflate avail_in=%d\n", strm->avail_in);
        out2_len = sizeof(out2) - strm->avail_out;
        fprintf(stderr, "%zd -> %zd r=%d\n", out1_len, out2_len, r);
        fprintf(stderr, "avail_in=%zd avail_out=%zd\n", strm->avail_in, strm->avail_out);
        assert(r == ISAL_DECOMP_OK);
        assert(strm->block_state == ISAL_BLOCK_FINISH);
        assert(IN_DATA_SIZE == out2_len);
        assert(strm->avail_in == 8);
    }
}

%] ./mytest

level=-1 nowrap=1
128 -> 122 r=1
pre isal_inflate avail_in=130
post isal_inflate avail_in=2
122 -> 128 r=0
avail_in=2 avail_out=0
Assertion failed: (strm->avail_in == 8), function test2, file mytest.c, line 194.
zsh: abort      programs/mytest

[gbtucker]

It does but can end after last trailer bytes by using IGZIP_ZLIB_NO_HDR or IGZIP_GZIP_NO_HDR. We don't have a IGZIP_DEFLATE_NO_HDR equivalent yet.

    int nowrap = 0;
...
        strm->crc_flag = nowrap ? ISAL_DEFLATE : ISAL_ZLIB_NO_HDR_VER;
        strm->next_in = out1 + (nowrap ? 0 : 2);

$ gcc -o test test.c -lisal -lz && ./test
level=-1 nowrap=0
128 -> 127 r=1
pre isal_inflate avail_in=135
post isal_inflate avail_in=8
127 -> 128 r=0
avail_in=8 avail_out=0

[rhpvorderman]

The number of eaten bits is in strm->read_in_length. You can use that number to determine the amount of eaten bytes as well. In python-isal I do it with strm->read_in_length / 8 and that seems to work well (no off by one errors with quite extensive testing and production use).