doujiang24 / lua-resty-kafka

Lua kafka client driver for the Openresty based on the cosocket API
BSD 3-Clause "New" or "Revised" License
801 stars 274 forks source link

The memory will not be released, even if collectgarbage("collect") is executed #157

Open lzle opened 1 year ago

lzle commented 1 year ago

The memory will not be released, even if collectgarbage("collect") is executed. thanks!

location / {
    content_by_lua '
        local kafka_producer = require("resty.kafka.producer")

        local brokers = {
            { host = "10.104.67.26", port = 9092 },
            { host = "10.104.67.35", port = 9092 },
            { host = "10.104.67.47", port = 9092 }
        }

        local producer_config = {
                        producer_type   = "async",
                        max_retry       = 1,
                        batch_num       = 1000,
                        max_buffering   = 1024000,
                        }

        local producer = kafka_producer:new(brokers, producer_config, nil)

        local message = string.rep('a',  512)

        for i = 1, 1024000 do
            local ok, err = producer:send('original-kernel-log', nil, message .. i)
            if not ok then
               ngx.say("add err:", err)
            end
        end

        while true do
            if producer.ringbuffer.num == 0 then
                ngx.log(ngx.INFO, "collectgarbage")
                collectgarbage("collect")
                ngx.sleep(30)
            end
            ngx.sleep(1)
        end

        ngx.say("ok")
    ';
}
lzle commented 1 year ago

I found the problem, it was caused by the table, next is demo, the memory will not release.

local table_new = require("table.new")
function _M.rewrite()
    local queue = table_new(10240000, 0)

    local message = string.rep('a',  1024)
    for i = 1, 10240000 do
        queue[i] = message .. i
    end

    for i = 1, 10240000 do
        queue[i] = nil
    end

    ngx.log(ngx.INFO, 'collectgarbage')
    collectgarbage("collect")

    front_resp.quit_normal()
end