[FEATURE]: incremental fetch of stream output.

Thanks for the plugin as always @rcarriga !

I was looking into the possibility of fetching the output of stdout while the process is running, and I think I was able to come up with a relatively good solution.

I'd like to hear your opinion on this and possibly add this feature.

Proposal

It is as simple as changing the condition inside streams.lua as follows.

      -- not read_err
      -- -- n is specified, wait until buffer has n length but no need to wait for complete.set()
      -- -- n is not specified, then wait until complete.set()
      while not read_err and not complete.is_set() and (not n or #buffer < n) do

Example Usage

Here is an example code that uses this feature and as you can see in # outputs, the elapsed ms value when using stdout.read(100) changes incrementally opposed to when no n is provided stdout.read().

Please go to the bottom to see explanation of each function.

local nio = require("nio")

local function make_proc()
  local proc = nio.process.run({
    cmd = "ping",
    args = { "google.com" },
  })
  assert(proc, "proc is nil")
  return proc
end

local function test_no_read_n()
  local proc = make_proc()
  local start = vim.loop.hrtime()
  nio.run(function()
    nio.sleep(5 * 1000) -- wait 5 sec and kill ping
    proc.signal(15)
  end)
  local out = proc.stdout.read()
  local elapsed = (vim.loop.hrtime() - start) / 1000 / 1000
  vim.print(string.format([[[%8.2f ms] '%s']], elapsed, out))
end

local function test_hundred_bytes()
  local proc = make_proc()
  local start = vim.loop.hrtime()

  for i = 0, 9 do
    local out = proc.stdout.read(100)
    local elapsed = (vim.loop.hrtime() - start) / 1000 / 1000
    vim.print(string.format([[[%8.2f ms, i: %s] '%s']], elapsed, i, out))
  end
  proc.signal(15)
end

local function test_print_each_newline()
  local proc = make_proc()
  local start = vim.loop.hrtime()

  local buffer = ""
  local line_count = 0
  local iter_count = 0
  while true do
    local out = proc.stdout.read(100)
    if not out then
      break
    end
    iter_count = iter_count + 1
    buffer = buffer .. out
    while true do
      local cr = string.find(buffer, "\n")
      if not cr then
        break
      end
      line_count = line_count + 1
      local elapsed = (vim.loop.hrtime() - start) / 1000 / 1000
      local msg = [[[%8.2f ms, line: %s, iter: %s] '%s']]
      vim.print(string.format(msg, elapsed, line_count, iter_count, buffer:sub(1, cr - 1)))
      buffer = buffer:sub(cr + 1)
    end
    if line_count >= 10 then
      break
    end
  end
  proc.signal(15)
end

nio.run(function()
  vim.print("====== test_no_read_n ======")
  -- `stdout.read()` (with no n) will wait until the process is finished.
  -- So, we don't see any outputs until ping is done (5 sec).
  test_no_read_n()
  vim.print("====== test_hundred_bytes ======")
  -- `stdout.read(100)` will fetch 100 bytes from the buffer.
  -- This will output stdout **as soon as** there are more than 100 bytes from the last call.
  test_hundred_bytes()
  vim.print("====== test_print_each_newline ======")
  -- Same as above but the function has mechanism to cut the buffer at "\n" using string.find.
  -- This is meant to be implemented on the user side, but maybe it'd be nice if
  -- nio could add this functionality as it is relatively simple. Perhaps...
  -- `stdout.read(n: integer?, until: string?)` where
  -- -- `stdout.read()`: wait and flush all
  -- -- `stdout.read(100)`: flush 100 bytes at a time
  -- -- `stdout.read(100, "\n")`: flush every 100 bytes but only return at every "\n"
  -- -- `stdout.read(nil, "\n")`: ERROR, user must provide `n` as the best guess of the line length (which will vary between applications) for performance reasons.
  test_print_each_newline()
end)

Output

====== test_no_read_n ======
[ 4980.84 ms] 'PING google.com (xxx.xxx.xxx.xxx) 56(84) bytes of data.
64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=1 ttl=117 time=2.45 ms
64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=2 ttl=117 time=2.36 ms
64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=3 ttl=117 time=2.56 ms
64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=4 ttl=117 time=2.50 ms
64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=5 ttl=117 time=2.46 ms
'
====== test_hundred_bytes ======
[   37.84 ms, i: 0] 'PING google.com (xxx.xxx.xxx.xxx) 56(84) bytes of data.\n64 bytes from _____________________.net (xxx.'
[ 1009.14 ms, i: 1] 'xxx.xxx.xxx): icmp_seq=1 ttl=117 time=2.40 ms\n64 bytes from _____________________.net (xxx.xxx.xxx.xxx'
[ 2011.21 ms, i: 2] '): icmp_seq=2 ttl=117 time=2.47 ms\n64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_se'
[ 3013.33 ms, i: 3] 'q=3 ttl=117 time=2.51 ms\n64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=4 ttl=11'
[ 4014.43 ms, i: 4] '7 time=2.48 ms\n64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=5 ttl=117 time=2.4'
[ 6018.45 ms, i: 5] '4 ms\n64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=6 ttl=117 time=2.46 ms\n64 by'
[ 7020.63 ms, i: 6] 'tes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=7 ttl=117 time=2.51 ms\n64 bytes from _'
[ 8021.74 ms, i: 7] '____________________.net (xxx.xxx.xxx.xxx): icmp_seq=8 ttl=117 time=2.52 ms\n64 bytes from __________'
[ 9022.75 ms, i: 8] '_________.net (xxx.xxx.xxx.xxx): icmp_seq=9 ttl=117 time=2.46 ms\n64 bytes from _____________________'
[10024.73 ms, i: 9] '.net (xxx.xxx.xxx.xxx): icmp_seq=10 ttl=117 time=2.45 ms\n64 bytes from _____________________.net (xxx'
====== test_print_each_newline ======
[    7.11 ms, line: 1, iter: 1] 'PING google.com (xxx.xxx.xxx.xxx) 56(84) bytes of data.'
[ 1008.22 ms, line: 2, iter: 2] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=1 ttl=117 time=2.36 ms'
[ 2009.54 ms, line: 3, iter: 3] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=2 ttl=117 time=2.44 ms'
[ 3010.35 ms, line: 4, iter: 4] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=3 ttl=117 time=2.55 ms'
[ 4012.43 ms, line: 5, iter: 5] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=4 ttl=117 time=2.47 ms'
[ 6016.71 ms, line: 6, iter: 6] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=5 ttl=117 time=2.53 ms'
[ 6016.74 ms, line: 7, iter: 6] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=6 ttl=117 time=2.43 ms'
[ 7017.63 ms, line: 8, iter: 7] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=7 ttl=117 time=2.54 ms'
[ 8019.71 ms, line: 9, iter: 8] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=8 ttl=117 time=2.44 ms'
[ 9021.83 ms, line: 10, iter: 9] '64 bytes from _____________________.net (xxx.xxx.xxx.xxx): icmp_seq=9 ttl=117 time=2.50 ms'

nvim-neotest / nvim-nio