Zip file produced has 0 bytes

joaoreynolds commented 4 months ago

Without changing my codebase, this utility suddenly stopped working this week. All zip files are now 0 bytes (invalid). I've been banging my head against the wall for at least 20 hours now, so any help in advance is very appreciated.

Environment

Node 20.13.1
Streaming files into it using the AWS S3 SDK (v3)

Code

const archiver = require('archiver')
const {
  S3Client,
  GetObjectCommand
} = require('@aws-sdk/client-s3')

const config = require('config')

const client = new S3Client({ region: 'us-east-2' })

module.exports = async function createArchiveFromS3(filePaths){
  const zipArchive = archiver('zip')

  console.log(`number of files: ${filePaths.length}`)

  zipArchive.on('progress', function (args) {
    console.log('//////progress')
    console.log(args)
  })

  zipArchive.on('finish', function (args) {
    console.log('-------finished')
  })

  // wait for all stream request from s3 to be done
  const streamsPromises = filePaths.map(myFilePath => streamFileFromS3(myFilePath))
  const s3Streams = await Promise.all(streamsPromises)

  s3Streams.map((myS3Stream, index) => {
    zipArchive.append(myS3Stream, { name: filePaths[index] })
  })

  console.log('finalizing...')
  zipArchive.finalize()

  return zipArchive

}

async function streamFileFromS3(s3Key) {
  const command = new GetObjectCommand({
    Bucket: config.s3Bucket,
    Key: s3Key,
  })
  const response = await client.send(command)
  console.log('got one file from s3')
  return response.Body
}

Output

number of files: 13
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
got one file from s3
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/rk0lGYi2a1709048309719.jpeg
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/rJx0gGKj261709048309721.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/ry-AxGKj361709048309722.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/BJzClfFs3T1709048309723.png
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/Hyb8zKs3a1709048393227.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/rykHmKj2a1709048630694.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/SkqwQYsnT1709048673598.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/B1doXYonT1709048736347.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/Bkr0XFi2a1709048780690.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/ryozNKi2a1709048850607.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/H1u8VYih61709048911527.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/By5F4Yo361709048961920.pdf
appending file: org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/BJLjEYjna1709048990265.pdf
finalizing...
//////progress
{
  entries: { total: 13, processed: 1 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 2 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 3 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 4 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 5 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 6 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 7 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 8 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 9 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 10 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 11 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 12 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
//////progress
{
  entries: { total: 13, processed: 13 },
  fs: { totalBytes: 0, processedBytes: 0 }
}
-------finished

This is what the "entry" event gives me, so you can see it's receiving a stream for each file.

{
  name: 'org-5927a1e253ea6534838869a1/perm-65de01f5f18dd0498b296394/rk0lGYi2a1709048309719.jpeg',
  type: 'file',
  date: 2024-05-19T22:58:16.009Z,
  mode: 420,
  prefix: null,
  sourcePath: null,
  stats: false,
  sourceType: 'stream',
  namePrependSlash: false,
  linkname: null,
  store: false,
  comment: ''
}

I then pipe this through to ExpressJS. The alarming thing to me is the "progress" report. Any ideas what's going on?

joaoreynolds commented 4 months ago

Oh wow, this problem existed earlier I just didn't have any users report it earlier. Looks like a few weeks ago I introduced a package that imported buffer, which my best guess is causing problems. Hopefully this helps someone in the future 🤞

dragomir-parvanov-riskmethods commented 4 months ago

huh, can you try awaiting zip.finalize(), but might be not the solution, but definitely an improvement

archiverjs / node-archiver