EPIPE Error when uploading large file using nodejs

[soubhikchatterjee]

I was following this article to setup a nodejs server on my local machine (which has 16 gb memory and about 170gb free disk-space) and uploaded a 20 gb file, for the first couple of times the file got uploaded successfully, but after a while i started getting EPIPE error:

error FetchError: request to http://localhost:3200/upload failed, reason: write EPIPE
    at ClientRequest.<anonymous> (/Volumes/FreeAgent GoFlex Drive/Test/multer-project/node_modules/node-fetch/lib/index.js:1455:11)
    at ClientRequest.emit (events.js:327:22)
    at Socket.socketErrorListener (_http_client.js:467:9)
    at Socket.emit (events.js:315:20)
    at emitErrorNT (internal/streams/destroy.js:100:8)
    at emitErrorCloseNT (internal/streams/destroy.js:68:3)
    at processTicksAndRejections (internal/process/task_queues.js:84:21) {
  type: 'system',
  errno: 'EPIPE',
  code: 'EPIPE'
}

When i checked, the file got uploaded partially and was about 28mb in size. I tried uploading the file from both Postman, browser and a nodejs script, but got the same EPIPE error message. I am not sure why is this happening, googling the error message didn't help. I am not sure how to overcome this. Following is my server and client code.

// server.js
const express = require("express"); // Express Web Server
const busboy = require("connect-busboy"); // Middleware to handle the file upload https://github.com/mscdex/connect-busboy
const path = require("path"); // Used for manipulation with path
const fs = require("fs-extra");  

const app = express(); // Initialize the express web server
app.use(
  busboy({
    highWaterMark: 2 * 1024 * 1024 // Set 2MiB buffer
  })
); // Insert the busboy middle-ware

const uploadPath = path.join(__dirname, "uploads/"); // Register the upload path
fs.ensureDir(uploadPath); // Make sure that he upload path exits

/**
 * Create route /upload which handles the post request
 */
app.route("/upload").post((req, res, next) => {
  req.pipe(req.busboy); // Pipe it trough busboy

  req.busboy.on("file", (fieldname, file, filename) => {
    console.log(`Upload of '${filename}' started`);

    // Create a write stream of the new file
    const fstream = fs.createWriteStream(path.join(uploadPath, filename));
    // Pipe it trough
    file.pipe(fstream);

    // On finish of the upload
    fstream.on("close", () => {
      console.log(`Upload of '${filename}' finished`);
      res.send("ok");
    });
  });
});

/**
 * Serve the basic index.html with upload form
 */
app.route("/").get((req, res) => {
  res.writeHead(200, { "Content-Type": "text/html" });
  res.write(
    '<form action="upload" method="post" enctype="multipart/form-data">'
  );
  res.write('<input type="file" name="fileToUpload"><br>');
  res.write('<input type="submit">');
  res.write("</form>");
  return res.end();
});

const server = app.listen(3200, function() {
  console.log(`Listening on port ${server.address().port}`);
});

and my client code is:

// client.js
const fs = require("fs");
const FormData = require("form-data");
const fetch = require("node-fetch");

var formdata = new FormData();
formdata.append(
  "file",
  fs.createReadStream("/Users/phantom007/My Documents/35gb.myfile")
);

var requestOptions = {
  method: "POST",
  body: formdata,
  redirect: "follow"
};

fetch("http://localhost:3200/upload", requestOptions)
  .then(response => response.text())
  .then(result => console.log(result))
  .catch(error => console.log("error", error));

[mscdex]

The server is somehow severing the connection abruptly. I don't see anything wrong with the code you've shown.

[soubhikchatterjee]

Exactly, I am serving the server on my localmachine, i dont see a reason why would it fail. How do i debug this? Pls help

[soubhikchatterjee]

anyone?

[soubhikchatterjee]

anyone in 2021?

[mscdex]

If you can reproduce this with the latest version of busboy then please post an issue to the busboy issue tracker.