mscdex / ssh2-streams

SSH2 and SFTP client/server protocol streams for node.js
MIT License
204 stars 143 forks source link

Node emits MaxListenersExceededWarning when downloading several thousand files with sftp fastget #96

Open watery opened 6 years ago

watery commented 6 years ago

This could be caused from my own code of course.

I'm downloading all the files from a remote server folder with sftp.fastGet(). Everything runs fine when working with roughly a hundred downloads.

When using a folder with around 40000 files Node emits this warning:

MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 drain listeners added. Use emitter.setMaxListeners() to increase limit
    at _addListener (events.js:280:19)
    at Deflate.addListener (events.js:297:10)
    at Deflate.Readable.on (_stream_readable.js:772:35)
    at Deflate.once (events.js:341:8)
    at Deflate.flush (zlib.js:346:12)
    at send (C:\Users\radaemarco\Documents\node\mongodb-news-container\src\retriever\node_modules\ssh2-streams\lib\ssh.js:5269:14)
    at SSH2Stream.channelData (C:\Users\radaemarco\Documents\node\mongodb-news-container\src\retriever\node_modules\ssh2-streams\lib\ssh.js:941:10)
    at Channel._write (C:\Users\radaemarco\Documents\node\mongodb-news-container\src\retriever\node_modules\ssh2\lib\Channel.js:345:21)
    at doWrite (_stream_writable.js:387:12)
    at writeOrBuffer (_stream_writable.js:373:5)

Here's a stripped down version of my code:

var main = function () {
  connection = new _ssh.Client();

  connection.on('ready', () => {
    connection.sftp((err, sftp) => {
      if(err) throw err;

      // whole folder reading
      sftp.readdir(_config.remote.folder, function (err, list) {
        if(err) {
          throw err;
        }

        processEntries(sftp, list)
      });
    })
  });

  connection.connect({ host: _config.access.host, username: _config.access.username, ...})
}();

function processEntries(sftp, list) {
  if(list instanceof Array) {
    list.forEach((listEntry, idx) => {
      let entryInfo = {
        name: _path.basename(listEntry.filename),
        creationTimeMs: listEntry.attrs.mtime * 1000,
        creationTime: new Date(parseInt(listEntry.attrs.mtime * 1000))
      }

      downloadFile(sftp, entryInfo);
    });
  }
}

function downloadFile(sftp, fileInfo) {
  let remoteFile = _config.remote.folder + "/" + fileInfo.name;
  let localFile = _path.resolve(_config.local.folder, fileInfo.name);

  sftp.fastGet(remoteFile, localFile, { concurrency: 1 }, err => {
    if(err) {
      throw err;
    }
  });

}

Actually, the downloadFile() is controlled by a database query result so there's potentially a callback / promise step in between that's not shown here.

Moreover, I tried adding all the downloadFile() in a chain of promises:

let chain = []
chain.push(
  (database generated Promise)
  .then(() => {
    downloadFile(sftp, info.info);
  })
);
});

chain.reduce((p, f) => p.then(f), Promise.resolve());

but I'm still get that warning.

Any hint? Am I doing something wrong?

mscdex commented 6 years ago

Node version?

watery commented 6 years ago

Node 8.9.4 Windows 7 Remote server is an AIX one