How would you handle large file uploads with streams in Node.js?

Direct answer

Pipe req (IncomingMessage) through size-limited transform or busboy/multer to fs.createWriteStream — never buffer entire file in memory; validate MIME/extension, scan path, return 413 if over limit; use pipeline for error cleanup.

Scenario: accept 100MB CSV import without OOM — stream from socket to disk or S3-compatible storage.

Streaming write — req.pipe(writeStream) or pipeline(req, limiter, writeStream).
Size cap — count bytes in Transform; destroy stream with 413 if exceeded.
Security — random filename, path.resolve check (Q66), virus scan async job.
Progress — optional job queue for post-upload processing via BullMQ.

scenario-upload-stream.mjs

import { pipeline } from "node:stream/promises";
import fs from "node:fs";
import { Transform } from "node:stream";

const MAX = 100 * 1024 * 1024;

function sizeLimiter(max) {
  let bytes = 0;
  return new Transform({
    transform(chunk, _enc, cb) {
      bytes += chunk.length;
      if (bytes > max) cb(new Error("FILE_TOO_LARGE"));
      else cb(null, chunk);
    },
  });
}

async function handleUpload(req, destPath) {
  await pipeline(
    req,
    sizeLimiter(MAX),
    fs.createWriteStream(destPath)
  );
}