egoist / tsup

The simplest and fastest way to bundle your TypeScript libraries.
https://tsup.egoist.dev
MIT License
9.17k stars 220 forks source link

tsup build failing with ERR_WORKER_OUT_OF_MEMORY #920

Open tigawanna opened 1 year ago

tigawanna commented 1 year ago

tsup build is failing on me when dts is enabled

cli output

> shadcn-fe-components@0.0.1 build C:\Users\denni\Desktop\code\workspace\shadcn-ui-fanedition\packages\ui
> tsup

CLI Building entry: src/index.ts, src/components/alert/index.ts, src/components/alert-dialog/index.ts, src/components/accordion/index.ts, src/components/aspect-ratio/index.ts, src/components/avatar/index.ts, src/components/badge/index.ts, src/components/button/index.ts, src/components/calendar/index.ts, src/components/card/index.ts, src/components/checkbox/index.ts, src/components/collapsible/index.ts, src/components/command/index.ts, src/components/context-menu/index.ts, src/components/dialog/index.ts, src/components/dropdown-menu/index.ts, src/components/hover-card/index.ts, src/components/input/index.ts, src/components/label/index.ts, src/components/menubar/index.ts, src/components/popover/index.ts, src/components/navigation-menu/index.ts, src/components/progress/index.ts, src/components/radio-group/index.ts, src/components/scroll-area/index.ts, src/components/select/index.ts, src/components/separator/index.ts, src/components/sheet/index.ts, src/components/skeleton/index.ts, src/components/slider/index.ts, src/components/switch/index.ts, src/components/table/index.ts, src/components/tabs/index.ts, src/components/textarea/index.ts, src/components/toast/index.ts, src/components/toggle/index.ts, src/components/tooltip/index.ts
CLI Using tsconfig: tsconfig.json
CLI tsup v6.7.0
CLI Using tsup config: C:\Users\denni\Desktop\code\workspace\shadcn-ui-fanedition\packages\ui\tsup.config.ts
CLI Target: esnext
CLI Cleaning output folder
ESM Build start
DTS Build start
"useCallback" is imported from external module "react" but never used in "dist/chunk-DNB5WMWA.js".
ESM dist\components\textarea\index.js            617.00 B
ESM dist\components\tabs\index.js                1.27 KB
ESM dist\components\toggle\index.js              1.08 KB
ESM dist\components\separator\index.js           516.00 B
ESM dist\components\skeleton\index.js            287.00 B
ESM dist\components\tooltip\index.js             786.00 B
ESM dist\components\select\index.js              2.30 KB
ESM dist\components\sheet\index.js               3.41 KB
ESM dist\components\scroll-area\index.js         1001.00 B
ESM dist\components\slider\index.js              884.00 B
ESM dist\components\table\index.js               1.64 KB
ESM dist\components\input\index.js               678.00 B
ESM dist\components\label\index.js               513.00 B
ESM dist\components\popover\index.js             776.00 B
ESM dist\components\navigation-menu\index.js     3.03 KB
ESM dist\components\progress\index.js            579.00 B
ESM dist\components\calendar\index.js            1.81 KB
ESM dist\components\radio-group\index.js         960.00 B
ESM dist\components\card\index.js                1.14 KB
ESM dist\components\checkbox\index.js            853.00 B
ESM dist\components\collapsible\index.js         256.00 B
ESM dist\components\menubar\index.js             4.21 KB
ESM dist\components\hover-card\index.js          615.00 B
ESM dist\components\command\index.js             2.92 KB
ESM dist\components\context-menu\index.js        3.57 KB
ESM dist\chunk-Q37AVSUX.js                       2.28 KB
ESM dist\components\dialog\index.js              308.00 B
ESM dist\components\dropdown-menu\index.js       3.85 KB
ESM dist\components\alert\index.js               1.10 KB
ESM dist\index.js                                356.00 B
ESM dist\components\alert-dialog\index.js        2.22 KB
ESM dist\components\accordion\index.js           257.00 B
ESM dist\chunk-B2EBCAU5.js                       1.13 KB
ESM dist\chunk-4Y755WKG.js                       1.39 KB
ESM dist\components\avatar\index.js              816.00 B
ESM dist\chunk-DNB5WMWA.js                       1.73 KB
ESM dist\components\button\index.js              222.00 B
ESM dist\components\badge\index.js               877.00 B
ESM dist\chunk-GUWPUQGY.js                       980.00 B
ESM dist\chunk-MVYCDPAD.js                       778.00 B
ESM dist\components\aspect-ratio\index.js        161.00 B
ESM dist\chunk-72SGZM6M.js                       20.12 KB
ESM dist\components\switch\index.js              960.00 B
ESM dist\components\toast\index.js               3.04 KB
ESM dist\components\textarea\index.js.map        1.14 KB
ESM dist\components\toggle\index.js.map          2.45 KB
ESM dist\components\separator\index.js.map       1.32 KB
ESM dist\components\skeleton\index.js.map        612.00 B
ESM dist\components\tooltip\index.js.map         1.60 KB
ESM dist\components\select\index.js.map          5.64 KB
ESM dist\components\sheet\index.js.map           9.47 KB
ESM dist\components\table\index.js.map           4.32 KB
ESM dist\components\label\index.js.map           1.30 KB
ESM dist\components\navigation-menu\index.js.map 6.94 KB
ESM dist\components\progress\index.js.map        1.32 KB
ESM dist\components\calendar\index.js.map        3.51 KB
ESM dist\components\tabs\index.js.map            2.67 KB
ESM dist\components\card\index.js.map            3.09 KB
ESM dist\components\checkbox\index.js.map        1.59 KB
ESM dist\components\radio-group\index.js.map     2.22 KB
ESM dist\components\collapsible\index.js.map     610.00 B
ESM dist\components\menubar\index.js.map         10.54 KB
ESM dist\components\hover-card\index.js.map      1.45 KB
ESM dist\components\command\index.js.map         7.19 KB
ESM dist\components\slider\index.js.map          1.65 KB
ESM dist\chunk-Q37AVSUX.js.map                   5.56 KB
ESM dist\components\alert\index.js.map           2.55 KB
ESM dist\components\dropdown-menu\index.js.map   9.62 KB
ESM dist\components\dialog\index.js.map          51.00 B
ESM dist\index.js.map                            51.00 B
ESM dist\components\accordion\index.js.map       51.00 B
ESM dist\components\scroll-area\index.js.map     2.53 KB
ESM dist\chunk-4Y755WKG.js.map                   5.73 KB
ESM dist\chunk-DNB5WMWA.js.map                   9.64 KB
ESM dist\components\avatar\index.js.map          2.20 KB
ESM dist\components\input\index.js.map           1.21 KB
ESM dist\components\button\index.js.map          51.00 B
ESM dist\components\badge\index.js.map           1.91 KB
ESM dist\components\aspect-ratio\index.js.map    364.00 B
ESM dist\chunk-GUWPUQGY.js.map                   1.88 KB
ESM dist\chunk-MVYCDPAD.js.map                   3.94 KB
ESM dist\chunk-72SGZM6M.js.map                   117.68 KB
ESM dist\components\switch\index.js.map          1.62 KB
ESM dist\components\toast\index.js.map           6.63 KB
ESM dist\components\popover\index.js.map         1.65 KB
ESM dist\components\context-menu\index.js.map    9.22 KB
ESM dist\components\alert-dialog\index.js.map    6.79 KB
ESM dist\chunk-B2EBCAU5.js.map                   3.01 KB
ESM ⚡️ Build success in 17910ms
node:events:489
      throw er; // Unhandled 'error' event
      ^

Error [ERR_WORKER_OUT_OF_MEMORY]: Worker terminated due to reaching memory limit: JS heap out of memory
    at new NodeError (node:internal/errors:399:5)
    at [kOnExit] (node:internal/worker:310:26)
    at Worker.<computed>.onexit (node:internal/worker:226:20)
Emitted 'error' event on Worker instance at:
    at [kOnExit] (node:internal/worker:310:12)
    at Worker.<computed>.onexit (node:internal/worker:226:20) {
  code: 'ERR_WORKER_OUT_OF_MEMORY'
}

Node.js v20.1.0
 ELIFECYCLE  Command failed with exit code 1.
import { defineConfig } from 'tsup'

export default defineConfig({
    dts: true,
    minify: true,
    sourcemap: true,
    treeshake: true,
    splitting: true,
    clean: true,
    external: ['react', 'react-dom'],
    entry: [
        "src/index.ts",
        "src/components/**/index.ts",
    ],
    format: ['esm'],
})

Upvote & Fund

Fund with Polar

RomainGueffier commented 1 year ago

Same with Tsup version 7.0.0 (Using pnpm, turbo) It happens only on DTS part. I also tried without turbo but same error. For really small library, it can complete without error but DTS build takes a really long time. (for example 500ms for esm build then 20+ s for DTS)

I have to rollback to Tsup 6.5.0 to make it work

Context: React library Tsup conf:

import { defineConfig } from 'tsup'

export default defineConfig((options) => {
  return {
    entry: ['src'],
    sourcemap: true,
    minify: !options.watch,
    dts: false,
    clean: true,
    format: ['esm'],
    external: ['react', 'react-hook-form'],
    // https://github.com/shuding/react-wrap-balancer/blob/main/tsup.config.ts#L10-L13
    esbuildOptions(options) {
      options.banner = {
        js: '"use client"',
      }
    },
  }
})

Terminal output on build script:

ESM ⚡️ Build success in 504ms
build: DTS Build start
build: node:events:491
build:       throw er; // Unhandled 'error' event
build:       ^
build: 
build: Error [ERR_WORKER_OUT_OF_MEMORY]: Worker terminated due to reaching memory limit: JS heap out of memory
build:     at new NodeError (node:internal/errors:393:5)
build:     at [kOnExit] (node:internal/worker:277:26)
build:     at Worker.<computed>.onexit (node:internal/worker:199:20)
build: Emitted 'error' event on Worker instance at:
build:     at [kOnExit] (node:internal/worker:277:12)
build:     at Worker.<computed>.onexit (node:internal/worker:199:20) {
build:   code: 'ERR_WORKER_OUT_OF_MEMORY'
build: }
build: 
build: Node.js v18.10.0
build:  ELIFECYCLE  Command failed with exit code 1.
build: ERROR: command finished with error: command (/srv/projects/unic/unics/customers-sites/customers-sites/packages/molecules) pnpm run build exited (1)
command (/srv/projects/customers-sites/packages/molecules) pnpm run build exited (1)
tigawanna commented 1 year ago

Luckily they have a build function

import glob from "glob"
import { build } from 'tsup'
import _ from 'lodash';

async function buildStage({ clean, entry }) {
    console.log("🚀 ~ building entry ", entry)

    try {
        await build({
          dts: true,
          minify: true,
          sourcemap: true,
          treeshake: true,
          splitting: true,
          outDir: 'dist',
          clean,
          entry,
          external: ['react', 'react-dom'],
          format: ['esm', 'cjs'],
        //   outExtension({ format }) {
        //     return {
        //       js: `.${format}.js`,
        //     };
        //   },
        });
    } catch (error) {
        console.log("🚀 ~ error while building entries :", entry);
        console.log(error);
        throw error;
    }
}

export async function buildAllStages() {

    const root_file = glob.sync('src/index.ts');
    const files = glob.sync('src/components/**/index.ts');
    const chunkSize = 3;
    const chunks = _.chunk(files, chunkSize);
    // await buildStage({ clean:true, entry: chunks[0] });
    for await (const [index, chunk] of chunks.entries()) {
      console.log('🚀 ~ chnk === ', chunk);
        await buildStage({ clean:index===0, entry: chunk });
    }
    await buildStage({ clean:false, entry: root_file });
    //    await buildStage({ clean:true, entry: root_file });

}

export function invokeBuild(){

buildAllStages().then(()=>{
    console.log("🚀 ~ buildAllStages success");
}).catch((error)=>{
    console.log("🚀 ~ buildAllStages error === ", error);
})
}
invokeBuild()

fixing it might not be that hard , might try it later

rajat1saxena commented 1 year ago

Rolling back to version 6.6.0 seems to be working.

arkmech commented 1 year ago

Unable to migrate back because I need 7.0.0 for this feature https://github.com/egoist/tsup/pull/925 (keep 'use client'; directive in my components.

Same error

Error [ERR_WORKER_OUT_OF_MEMORY]: Worker terminated due to reaching memory limit: JS heap out of memory
arkmech commented 1 year ago

My config if helpful

import { defineConfig, Options } from 'tsup';

export default defineConfig((options: Options) => ({
  entry: {
    'Button/index': 'src/components/atoms/Button/Button.tsx',
    // lots more components
  },
  splitting: false,
  format: ['esm', 'cjs'],
  dts: true,
  minify: true,
  external: ['react'],
  ...options,
}));

On tsup ^7.2.0, completely blocked by this

arkmech commented 1 year ago

Is there any work around for this?

millerized commented 1 year ago

Rolling back to version 6.6.0 seems to be working.

Thanks @rajat1saxena.

I can confirm that something in v6.7.0 has created a performance regression -- both in speed and memory usage (OOM).

On my MBP (2.6 GHz 6-Core Intel Core i7, 64 GB DDR4) I'm seeing 10 second dts build times with v6.6.0 and 45 second dts build times with v6.7.0.

I have not measured memory usage yet but I am also seeing OOM in Github Actions.

9vfQbg7z4ajrGQxR commented 1 year ago

Same here, with this conf when watching for changes (--watch):

export default defineConfig((options: Options) => ({
  banner: {
    js: ' "use client";',
  },
  splitting: false,
  entry: [
    'src/*/index.ts',
  ],
  format: ['cjs'],
  dts: true,
  minify: isProduction,
  clean: isProduction,
  bundle: true,
  external: ['react', 'react-dom'],
  ...options,
}));

It's not systematic, but sometimes after a few minutes we're affected. version 6.6.0 doesn't have this problem. (as mentioned above)

juliobetta commented 1 year ago

I'm using 7.2.0. My temporary solution was putting NODE_OPTIONS='--max-old-space-size=16384' before tsup. it's a workaround, I know... but 😬

package.json

{
  "scripts": {
    "build": "NODE_OPTIONS='--max-old-space-size=16384' tsup",
    "dev": "NODE_OPTIONS='--max-old-space-size=16384' tsup --watch",
  }
}
toteto commented 12 months ago

Duplicate of #875

dalechyn commented 11 months ago
NODE_OPTIONS='--max-old-space-size=16384'

Thanks, worked magically in my case.

toteto commented 11 months ago
NODE_OPTIONS='--max-old-space-size=16384'

Thanks, worked magically in my case.

It works for you now but wait a bit while your project grows and it will start to fail. Also while it doesn't fail, it takes significant amount of time that will also cost via your CI.

TheMikeyRoss commented 10 months ago

My team is having the same issue here

Our tsup.config.ts here

and it seems the issue only happens when we run tsup with --dts flag

TheMikeyRoss commented 10 months ago

I tried running the build commands sequentially like this:


const { build } = require("tsup");

const {
  buildAllConfig,
  buildBlocksConfig,
  buildCoreConfig,
  buildElementsConfig
} = require("./tsup.config");

async function sequentialBuild() {
  await build(buildAllConfig);
  await build(buildCoreConfig);
  await build(buildBlocksConfig);
  await build(buildElementsConfig);
}

sequentialBuild().catch((err) => {
  console.error(err);
  process.exit(1);
});

where my tsup configurations are split like this:

export const buildAllConfig = defineConfig({
  name: "Build All",
  clean: true,
  dts: true,
  target: "es2019",
  entry: { index: "components/index.ts" },
  format: ["cjs", "esm"]
});
export const buildCoreConfig = defineConfig({
  name: "Build Core",
  clean: true,
  dts: true,
  target: "es2019",
  format: ["cjs", "esm"],
  entry: {
    // CORE
    "types/index": "components/types/index.ts",
    "hooks/index": "components/hooks/index.ts",
    "blocks/index": "components/blocks/index.ts",
    "layout/index": "components/layout/index.ts",
    "elements/index": "components/elements/index.ts"
  }
});
export const buildBlocksConfig = defineConfig({
  name: "Build Blocks",
  clean: true,
  dts: true,
  target: "es2019",
  format: ["cjs", "esm"],
  entry: {
    // BLOCKS
    "blocks/misc/index": "components/blocks/misc/index.ts",
    "blocks/auth/index": "components/blocks/auth/index.ts",
    "blocks/pricing/index": "components/blocks/pricing/index.ts",
    "blocks/feedback/index": "components/blocks/feedback/index.ts"
  }
});
export const buildElementsConfig = defineConfig({
  name: "Build Elements",
  clean: true,
  dts: true,
  target: "es2019",
  format: ["cjs", "esm"],
  entry: {
    // ELEMENTS
    "card/index": "components/elements/card/index.ts",
    "chip/index": "components/elements/chip/index.ts",
    "tabs/index": "components/elements/tabs/index.ts",
    "sheet/index": "components/elements/sheet/index.ts",
    "logos/index": "components/elements/logos/index.ts",
    "radio/index": "components/elements/radio/index.ts",
    "table/index": "components/elements/table/index.ts",
    "alert/index": "components/elements/alert/index.ts",
    "label/index": "components/elements/label/index.ts",
    "input/index": "components/elements/input/index.ts",
    "badge/index": "components/elements/badge/index.ts",
    "dialog/index": "components/elements/dialog/index.ts",
    "button/index": "components/elements/button/index.ts",
    "select/index": "components/elements/select/index.ts",
    "avatar/index": "components/elements/avatar/index.ts",
    "switch/index": "components/elements/switch/index.ts",
    "command/index": "components/elements/command/index.ts",
    "popover/index": "components/elements/popover/index.ts",
    "loading/index": "components/elements/loading/index.ts",
    "tooltip/index": "components/elements/tooltip/index.ts",
    "skeleton/index": "components/elements/skeleton/index.ts",
    "combobox/index": "components/elements/combobox/index.ts",
    "textarea/index": "components/elements/textarea/index.ts",
    "pinInput/index": "components/elements/pinInput/index.ts",
    "checkbox/index": "components/elements/checkbox/index.ts",
    "progress/index": "components/elements/progress/index.ts",
    "accordion/index": "components/elements/accordion/index.ts",
    "backToTop/index": "components/elements/backToTop/index.ts",
    "dataTable/index": "components/elements/dataTable/index.ts",
    "appStores/index": "components/elements/appStores/index.ts",
    "sortButton/index": "components/elements/sortButton/index.ts",
    "scrollArea/index": "components/elements/scrollArea/index.ts",
    "breadcrumb/index": "components/elements/breadcrumb/index.ts",
    "phoneInput/index": "components/elements/phoneInput/index.ts",
    "splitButton/index": "components/elements/splitButton/index.ts",
    "dropdownMenu/index": "components/elements/dropdownMenu/index.ts",
    "fileDropzone/index": "components/elements/fileDropzone/index.ts",
    "navigationMenu/index": "components/elements/navigationMenu/index.ts",
    "stopPropagationWrapper/index":
      "components/elements/stopPropagationWrapper/index.ts"
  }
});

But I'm still getting the same error:

Error [ERR_WORKER_OUT_OF_MEMORY]: Worker terminated due to reaching memory limit: JS heap out of memory
tigawanna commented 10 months ago

Try only building 3 files at a time

TheMikeyRoss commented 10 months ago

Try only building 3 files at a time

That actually worked! thanks so much @tigawanna

krishSagar commented 10 months ago

Thanks @tigawanna your workaround worked. I was able to build up to 6 files at a time without issues. However, I grouped my files according to their categories

import { defineConfig } from 'tsup'

export const fonts = defineConfig({
    entry: {
        fonts: 'src/styles/fonts.ts',
        commonTypes: 'src/CommonTypes/index.ts'

    }
})
export const maps = defineConfig({
    entry: {
        lib: 'src/lib/index.ts',
        charts: 'src/PgCharts/index.ts',
        error: 'src/ErrorComponent/index.ts'
    }
})
export const forms = defineConfig({
    entry: {
        pgForm: 'src/pgForm/index.ts',
        pgForm2: 'src/pgForm2/index.ts'
    }
})

export const icons = defineConfig({
    entry: {
        icons: 'src/PgIcon/index.ts'
    }
})

export const tables = defineConfig({
    entry: {
        pgTable: 'src/PgTable/index.ts'
    }
})
export const map = defineConfig({
    entry: {
        pgMap: 'src/PgMap/index.ts'
    }
})
export const dialogs = defineConfig({
    entry: {
        alert: 'src/PgAlert/index.ts',
        popOver: 'src/pgPopOver/index.ts',
        card: 'src/PgCard/index.ts'
    }
})
export const jsonToCsv = defineConfig({
    entry: {
        jsonToCsv: 'src/JsonToCsv/index.ts'
    }
})
export const translations = defineConfig({
    entry: {
        translations: 'src/translations/index.ts'
    }
})
export const heatmap = defineConfig({
    entry: {
        heatmap: 'src/PgHeatmap/index.ts'
    }
})
export const utils = defineConfig({
    entry: {
        utils: 'src/CommonUtils/index.ts',
        formattedNumberInput: 'src/FormattedNumberInput/index.ts'
    }
})
export const styles = defineConfig({
    entry: {
        styles: 'src/styles/index.ts'
    }
})

export default [styles, utils, translations, heatmap, jsonToCsv, dialogs, map, tables, icons, forms, maps, fonts]

Such grouping reduced the number of files built in a set and also made the code look a bit organised. It does rids us of this ERR_WORKER_OUT_OF_MEMORY error until a permanent fix is issued from @egoist

tigawanna commented 10 months ago

Glad it helped the fix might be spawning a new worker for every ~4 files , but it feels like such a niche use case that the workaround will be fine for most

9vfQbg7z4ajrGQxR commented 10 months ago

Any news @egoist on this point ? it's preventing us from upgrading to the latest tsup version :disappointed: (we are stuck on 6.x)

maidul98 commented 9 months ago

Just faced the same issue. Would love if someone from the tsup team can look into patching this

harry-gocity commented 8 months ago

Getting the same issue on a library with ~100 entrypoints. Running locally I have no problems but in CI seeing the OOM error.

Splitting the config into chunks of 2,4,6 basically grinds the whole process to a halt locally. Interestingly experimentalDts has no memory problems at all, however does generate slightly incorrect declaration files for us, so not an option.

iamnnort commented 7 months ago

I have the same issue with "tsup": "^7.2.0" and 24 entry files.

jonstuebe commented 6 months ago

I played around with a chunking approach where the rollup.js worker file will chunk the inputs (configurable via a tsup option), however, this isn't possible in watch mode as watch mode is happening via rollup's watch mode. I believe tsup would need to take over watching itself using something like chokidar. You can see my adjusted runRollup function below:

async function runRollup(options: RollupConfig, chunkSize: number) {
  const { rollup } = await import('rollup')
  try {
    const start = Date.now()
    const getDuration = () => {
      return `${Math.floor(Date.now() - start)}ms`
    }
    logger.info('dts', 'Build start')

    if (!options.inputConfig.input) {
      logger.error('dts', 'No input')
      return
    }

    for (const input of chunkInput(options.inputConfig.input, chunkSize)) {
      const inputConfig = {
        ...options.inputConfig,
        input,
      }

      const bundle = await rollup(inputConfig)
      const results = await Promise.all(options.outputConfig.map(bundle.write))
      const outputs = results.flatMap((result) => result.output)
      logger.success('dts', `⚡️ Chunk build success in ${getDuration()}`)
      reportSize(
        logger,
        'dts',
        outputs.reduce((res, info) => {
          const name = path.relative(
            process.cwd(),
            path.join(options.outputConfig[0].dir || '.', info.fileName)
          )
          return {
            ...res,
            [name]:
              info.type === 'chunk' ? info.code.length : info.source.length,
          }
        }, {})
      )
    }
  } catch (error) {
    handleError(error)
    logger.error('dts', 'Build error')
  }
}

What do y'all think?

nolde commented 5 months ago

Using tsup@8.0.2 with --dts gives me the memory issue, but using --experimental-dts does not, and types seem to be correctly set up.

wujekbogdan commented 5 months ago

I'm having the same issue with the following tsup config:

defineConfig({
  entry: ['src/index.ts'],
  target: 'es2022',
  format: ['cjs', 'esm'],
  clean: true,
  sourcemap: true,
  dts: false,
});

and the following TypeScript config:

{
  "$schema": "https://json.schemastore.org/tsconfig",
  "display": "Default",
  "compilerOptions": {
    "outDir": "dist",
    "lib": ["ES2022", "dom"],
    "module": "ESNext",
    "target": "ES2022",
    "composite": false,
    "emitDeclarationOnly": true,
    "declaration": true,
    "declarationMap": true,
    "esModuleInterop": true,
    "forceConsistentCasingInFileNames": true,
    "inlineSources": false,
    "isolatedModules": true,
    "moduleResolution": "node",
    "noUnusedLocals": false,
    "noUnusedParameters": false,
    "preserveWatchOutput": true,
    "skipLibCheck": true,
    "strict": true,
    "resolveJsonModule": true
  },
  "exclude": ["node_modules", "dist"]
}

I noticed that if I set treeshake to true, memory utilization goes even higher. I haven't measured it, but my pipeline fails much earlier when it's enabled.

I guess it has something to do with the fact that treeshake: true makes tsup use rollup rather than esbuild [source]

I also wonder how Turborepo contributes to memory utilization. Is it OK to spawn so many tsup/esbuild processes at a time? Is it a use case that tsup developers take into account?


nolde commented 5 months ago

I have been using onSuccess hook to just use tsc to compile the types, while tsup takes care of code. This took any memory issues away, and allowed me to build .d.ts files for all internal files, allowing deep lib reference with exports.

This approach was better than --experimental-dts as the types are easier to use in editors.

// tsup.config.ts
import { copyFile } from 'node:fs/promises'
import { exec } from 'node:child_process'
import { promisify } from 'node:util'

import glob from 'tiny-glob'
import { defineConfig } from 'tsup'

const pexec = promisify(exec)

export default defineConfig({
  cjsInterop: true,
  clean: true,
  entry: ['src/**/*.ts', '!src/**/*.test.ts'],
  format: ['cjs', 'esm'],
  shims: true,
  sourcemap: false,
  splitting: true,
  target: 'node20',
  //
  async onSuccess () {
    try {
      await pexec('tsc --emitDeclarationOnly --declaration')
      const files = await glob('dist/**/*.d.ts')
      await Promise.all(files.map(file => copyFile(file, file.replace('.d.ts', '.d.mts')))) // or to `.d.cjs` for `"type": "module"` projects
    } catch (err) {
      console.error()
      console.error('Typescript compilation error:')
      console.error()
      console.error(err.stdout)
      throw err
    }
  }
})

Don't forget to add "outDir": "./dist" to your tsconfig.

wujekbogdan commented 5 months ago

I doubt that .d.ts generation is the only issue here. In my case, having dts: false still causes memory issues. I found that removing the treeshake: true property helps a little because then my pipeline survives longer before failing, but there seems to be some general memory leak that is independent of all these settings. I suppose that some of the settings just contribute to the leak, but aren't the main culprit.

tysonclugg commented 4 months ago

Can someone please confirm if this issue still exists when using Node.js v21 or above?

This may have been fixed upstream in Node.js as per https://github.com/nodejs/node/issues/25382 in https://github.com/nodejs/node/commit/ce4102e8a559b4f36ee674ef92ddb046b45db789 (part of Node from v21.0.0 onwards), which changes the way that heap memory is allocated to forked child threads.

eweseongyeoh commented 4 months ago

Hi, we have observed that the latest v21 is slightly better as it was able to compute more folders but it eventually encounter the same issue.

Error [ERR_WORKER_OUT_OF_MEMORY]: Worker terminated due to reaching memory limit: JS heap out of memory
    at [kOnExit] (node:internal/worker:313:26)
    at Worker.<computed>.onexit (node:internal/worker:229:20)
Emitted 'error' event on Worker instance at:
    at [kOnExit] (node:internal/worker:313:12)
    at Worker.<computed>.onexit (node:internal/worker:229:20) {
  code: 'ERR_WORKER_OUT_OF_MEMORY'
}

Node.js v21.7.3

<--- Last few GCs --->
toteto commented 2 months ago

Still the case with Node 22.7.0

  DTS Build start
  node:events:498
        throw er; // Unhandled 'error' event
        ^

  Error [ERR_WORKER_OUT_OF_MEMORY]: Worker terminated due to reaching memory limit: JS heap out of memory
      at [kOnExit] (node:internal/worker:313:26)
      at Worker.<computed>.onexit (node:internal/worker:229:20)
  Emitted 'error' event on Worker instance at:
      at [kOnExit] (node:internal/worker:313:12)
      at Worker.<computed>.onexit (node:internal/worker:229:20) {
    code: 'ERR_WORKER_OUT_OF_MEMORY'
  }
DaveKeehl commented 4 weeks ago

@egoist any update on this? I'm also heavily blocked

cabljac commented 3 weeks ago

bumping this thread, we're running into this problem on certain builds in Firebase Genkit

DaveKeehl commented 3 weeks ago

I decided to run tsup multiple times with a small batch size to avoid the out-of-memory (OOM) error. Below is the content of build.js, a script I use to build my React component library. This approach helps prevent OOM errors both locally and in CI.

import { exec } from "child_process";
import fs from "fs";
import path from "path";

const componentsDir = "src/components";
const batchSize = 3;
const tsupFlags =
  "--format esm --tsconfig tsconfig.app.json --external react --external react-dom --external class-variance-authority --external clsx --external tailwind-merge --dts";

const components = fs
  .readdirSync(componentsDir)
  .filter((file) => fs.statSync(path.join(componentsDir, file)).isDirectory());

const buildBatches = (components) => {
  const batches = [];
  for (let i = 0; i < components.length; i += batchSize) {
    batches.push(components.slice(i, i + batchSize));
  }
  return batches;
};

const transpileComponents = (batch) => {
  const entryPoints = batch
    .map((comp) => `--entry ${path.join(componentsDir, comp, "index.tsx")}`)
    .join(" ");
  const command = `tsup ${entryPoints} --outDir dist/components ${tsupFlags}`;

  exec(command, (error, stdout, stderr) => {
    if (error) {
      console.error(`Error compiling batch: ${error.message}`);
      return;
    }

    if (stderr) {
      console.error(`stderr: ${stderr}`);
      return;
    }

    console.log(`stdout: ${stdout}`);
  });
};

const batches = buildBatches(components);
exec(`tsup src/index.ts ${tsupFlags}`);
batches.forEach(transpileComponents);
ale-vncs commented 2 weeks ago

I was same problem and fixed doing this:

tsup make build and tsc emit d.ts

// tsup.config.ts
export default defineConfig((options) => ({
  entry: ['src/**/*'],
  format: ['cjs', 'esm'],
  dts: false, // disabled d.ts IMPORTANT!
  sourcemap: true,
  clean: true,
  splitting: false,
  external: ['react', 'react-dom'],
  onSuccess: async () => {
    execSync('node generate-exports.js', { stdio: 'inherit' });
  },
  ...options,
}));
// tsconfig.json
{
  "compilerOptions": {
    "target": "es2018",
    "module": "ESNext",
    "jsx": "react-jsx",
    "moduleResolution": "Node",
    "outDir": "./dist",
    "declaration": true,
    "declarationMap": false,
    "emitDeclarationOnly": true, // Just emit declaration IMPORTANT
    "esModuleInterop": true,
    "skipLibCheck": true,
    "forceConsistentCasingInFileNames": true,
    "strict": true,
    "baseUrl": "src",
    "paths": {
      "@components/*": ["components/*"],
      "@styles/*": ["styles/*"],
      "@typings/*": ["typings/*"],
      "@utils/*": ["utils/*"]
    }
  },
  "include": ["src"]
}
// package.json
{
...
  "scripts": {
    "build": "tsup && tsc"
  }
}
wujekbogdan commented 2 weeks ago

@ale-vncs Sadly this solution doesn't work for everyone. It just reduces the memory footprint (leaks?), so it works in your case because you're not hitting the limit, but my project still fails even when I disable DTS generation. See: https://github.com/egoist/tsup/issues/920#issuecomment-2139202395