Upload file stream directly to S3. Unknown content-length

[AustinGil]

Hello. I'm trying to upload files from a multi-part/form-data request through my server and directly to S3. I'm piping the request to Busboy and trying to pipe each file to an S3 upload request, but S3 complains that the content-length is not defined.

import busboy from 'busboy';
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';

const s3Client = new S3Client({
  endpoint: process.env.S3_URL,
  credentials: {
    accessKeyId: process.env.S3_ACCESS_KEY,
    secretAccessKey: process.env.S3_SECRET_KEY,
  },
  region: process.env.S3_REGION,
});

/** @param {import('http').IncomingMessage} req */
function parseMultipartNodeRequest(req) {
  return new Promise((resolve, reject) => {
    const uploadRequests = [];

    const bb = busboy({ headers: req.headers });

    bb.on('file', (name, file, info) => {
      const uploadParams = {
        Bucket: 'bucket',
        Key: name,
        Body: file,
      };
      uploadRequests.push(s3Client.send(new PutObjectCommand(uploadParams));
    });
    bb.on('close', () => {
      Promise.all(uploadRequests).then(resolve);
    });
    req.pipe(bb);
  });
}

Is there a way to tell AWS what the content-length is without first storing the files on disk?

[AustinGil]

Ok, for some reason, passing the stream directly to S3 doesn't work, but using lib-storage does. Hopefully this helps folks:

import busboy from 'busboy';
import { Upload } from '@aws-sdk/lib-storage';
import { S3Client} from '@aws-sdk/client-s3';

const s3Client = new S3Client({
  endpoint: process.env.S3_URL,
  credentials: {
    accessKeyId: process.env.S3_ACCESS_KEY,
    secretAccessKey: process.env.S3_SECRET_KEY,
  },
  region: process.env.S3_REGION,
});

/** @param {import('http').IncomingMessage} req */
function parseMultipartNodeRequest(req) {
  return new Promise((resolve, reject) => {
    const uploadRequests = [];

    const bb = busboy({ headers: req.headers });

    bb.on('file', (name, file, info) => {
      const uploadParams = {
        Bucket: 'bucket',
        Key: name,
        Body: file,
      };
      const parallelUploads3 = new Upload({
        client: s3Client,
        params: uploadParams,
      });
      uploadRequests.push(parallelUploads3.done());
    });
    bb.on('close', () => {
      Promise.all(uploadRequests).then(resolve);
    });
    req.pipe(bb);
  });
}