Unable to patch executable produced by `deno compile` and use it in Nix. REPL starts instead of program.

[MrFoxPro]

I'm trying to build application with deno compile to single executable with Nix and run it in nix container:

packages.api = pkgs.stdenv.mkDerivation rec {
  name = "api";
  src = ./api;
  __noChroot = true;
  nativeBuildInputs = with pkgs; [
    deno
    unzip
    autoPatchelfHook
  ];
  buildInputs = with pkgs; [
    stdenv.cc.cc
  ];
  buildPhase = ''
    runHook preBuild
    export DENO_DIR=./.deno
    mkdir $DENO_DIR
    deno cache --reload --lock ./deno.lock --lock-write main.ts
    mkdir -p $out/bin
    deno compile -A --check --lock=deno.lock ./main.ts --cached-only --output $out/bin/${name} --target=x86_64-unknown-linux-gnu
    runHook postBuild
  '';
};

Patched libraries:

foxpro:~/craft/sferadel/next$ /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/ld-linux-x86-64.so.2 --list /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api                  19:22 27.07
        linux-vdso.so.1 (0x00007fffa8f36000)
        libdl.so.2 => /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/libdl.so.2 (0x00007ff7134a0000)
        libgcc_s.so.1 => /nix/store/m6kphh9rh424vbw5wq84jcg0r4is4sc5-gcc-12.3.0-libgcc/lib/libgcc_s.so.1 (0x00007ff71347f000)
        libpthread.so.0 => /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/libpthread.so.0 (0x00007ff71347a000)
        libm.so.6 => /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/libm.so.6 (0x00007ff71339a000)
        libc.so.6 => /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/libc.so.6 (0x00007ff70d21a000)
        /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/ld-linux-x86-64.so.2 (0x00007ff7134a7000)

Starting produced binary:

foxpro:~/craft/sferadel/next$ with-env {LD_DEBUG: files} { /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api }                                                                            19:23 27.07
    505344:
    505344:     file=libdl.so.2 [0];  needed by /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api [0]
    505344:     file=libdl.so.2 [0];  generating link map
    505344:       dynamic: 0x00007f26e71e3da8  base: 0x00007f26e71e0000   size: 0x0000000000004010
    505344:         entry: 0x00007f26e71e0000  phdr: 0x00007f26e71e0040  phnum:                 11
    505344:
    505344:
    505344:     file=libgcc_s.so.1 [0];  needed by /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api [0]
    505344:     file=libgcc_s.so.1 [0];  generating link map
    505344:       dynamic: 0x00007f26e71dec00  base: 0x00007f26e71bf000   size: 0x0000000000020148
    505344:         entry: 0x00007f26e71bf000  phdr: 0x00007f26e71bf040  phnum:                 10
    505344:
    505344:
    505344:     file=libpthread.so.0 [0];  needed by /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api [0]
    505344:     file=libpthread.so.0 [0];  generating link map
    505344:       dynamic: 0x00007f26e71bdda8  base: 0x00007f26e71ba000   size: 0x0000000000004010
    505344:         entry: 0x00007f26e71ba000  phdr: 0x00007f26e71ba040  phnum:                 11
    505344:
    505344:
    505344:     file=libm.so.6 [0];  needed by /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api [0]
    505344:     file=libm.so.6 [0];  generating link map
    505344:       dynamic: 0x00007f26e71b8c58  base: 0x00007f26e70da000   size: 0x00000000000df018
    505344:         entry: 0x00007f26e70da000  phdr: 0x00007f26e70da040  phnum:                 11
    505344:
    505344:
    505344:     file=libc.so.6 [0];  needed by /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api [0]
    505344:     file=libc.so.6 [0];  generating link map
    505344:       dynamic: 0x00007f26e70ca980  base: 0x00007f26e6ef4000   size: 0x00000000001e5d70
    505344:         entry: 0x00007f26e6f17c90  phdr: 0x00007f26e6ef4040  phnum:                 14
    505344:
    505344:
    505344:     calling init: /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/ld-linux-x86-64.so.2
    505344:
    505344:
    505344:     calling init: /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/libc.so.6
    505344:
    505344:
    505344:     calling init: /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/libm.so.6
    505344:
    505344:
    505344:     calling init: /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/libpthread.so.0
    505344:
    505344:
    505344:     calling init: /nix/store/m6kphh9rh424vbw5wq84jcg0r4is4sc5-gcc-12.3.0-libgcc/lib/libgcc_s.so.1
    505344:
    505344:
    505344:     calling init: /nix/store/1x4ijm9r1a88qk7zcmbbfza324gx1aac-glibc-2.37-8/lib/libdl.so.2
    505344:
    505344:
    505344:     initialize program: /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api
    505344:
    505344:
    505344:     transferring control: /nix/store/agib1vzv0i6br7cy7kdl1ymcm01w5wjx-api/bin/api
    505344:
Deno 1.35.2
exit using ctrl+d, ctrl+c, or close()
REPL is running with all permissions allowed.
To specify permissions, run `deno repl` with allow flags.
> 

Related: https://github.com/denoland/deno/issues/12780

[MrFoxPro]

I was able to build it like this:

packages.api = let
  target = "x86_64-unknown-linux-gnu";
  denoCache = pkgs.stdenv.mkDerivation {
    name = "api-deno-cache";
    src = ./api;
    nativeBuildInputs = with pkgs; [
      deno
      unzip
    ];
    configurePhase = ''
      echo "Configuring denoCache"
    '';
    buildPhase = ''
      export DENO_DIR=./.deno; mkdir $DENO_DIR
      deno cache --reload main.ts
    '';
    installPhase = ''
      mkdir $out
      cp -r .deno/deps $out
      cp -r .deno/npm $out
    '';
    outputHashMode = "recursive";
    outputHashAlgo = "sha256";
    outputHash = "sha256-8FzGaIEGM5RIJF0qAi9qKYtZyG+pPw+2kwIOQb171UE=";
    # outputHash = lib.fakeSha256;
  };
  compilier = pkgs.fetchurl {
    url = "https://dl.deno.land/release/v${pkgs.deno.version}/deno-${target}.zip";
    sha256 = "sha256-Q2AdVMRh5+NFhOhFDZKjII5Ol3D/3ebND4LP64HEqks=";
  };
in
  pkgs.stdenv.mkDerivation rec {
    name = "api";
    src = ./api;
    nativeBuildInputs = with pkgs; [
      deno
      unzip
      autoPatchelfHook
    ];
    buildInputs = with pkgs; [
      stdenv.cc.cc
    ];
    dontFixup = true;
    # dontStrip = true;

    configurePhase = ''
      echo "Built cache is on: ${denoCache}"
      export DENO_DIR=.deno
      mkdir $DENO_DIR
      ln -s ${denoCache}/.deno/deps $DENO_DIR/deps
      ln -s ${denoCache}/.deno/npm $DENO_DIR/npm

      mkdir -p $DENO_DIR/dl/release/v${pkgs.deno.version}
      ln -s ${compilier} $DENO_DIR/dl/release/v${pkgs.deno.version}/deno-${target}.zip
      ls -la $DENO_DIR/dl/release/v${pkgs.deno.version}/
    '';
    buildPhase = ''
      mkdir -p $out/bin
      deno compile --unstable -A --check --cached-only --lock deno.lock ./main.ts --output $out/bin/${name} --target=${target}
    '';
  };

[MrFoxPro]

But it doesn't work this way in Nix Containers. So this is related then: https://github.com/denoland/deno/issues/18764

I was able to bypass it with following hack:

packages.oci-api = nix2c.buildImage {
  name = "oci-api";
  config = {
    Cmd = ["${getExe self'.packages.api}"];
    WorkingDir = "/";
    Env = [
      "DENO_DIR=${self'.packages.api}/.deno"
      "DENO_INSTALL_ROOT=${pkgs.deno}"
      "DENO_VERSION=${pkgs.deno.version}"
      "LD_LIBRARY_PATH=${pkgs.stdenv.cc.libc_lib}/lib"
    ];
    ExposedPorts = {
      "${toString 3001}/tcp" = {};
    };
  };
  copyToRoot = [
    (pkgs.buildEnv {
      name = "root";
      paths = [
        pkgs.deno
        self'.packages.api
        pkgs.stdenv.cc.libc_lib
      ];
      pathsToLink = "/";
    })
  ];
};

This is still not a solution.

[MrFoxPro]

@06kellyjac

[MrFoxPro]

Reproduction:

[MrFoxPro]

Ok, with help of discord community I can show this:

pkgs.stdenv.mkDerivation rec {
  pname = "api";
  version = "0.0.0-rc1";
  src = ./api;
  __noChroot = true;
  compilier = pkgs.fetchurl {
    url = "https://dl.deno.land/release/v${pkgs.deno.version}/deno-${target}.zip";
    sha256 = "sha256-Q2AdVMRh5+NFhOhFDZKjII5Ol3D/3ebND4LP64HEqks=";
  };
  buildInputs = with pkgs; [stdenv.cc.cc];
  nativeBuildInputs = with pkgs; [deno unzip autoPatchelfHook unixtools.xxd hexdump];
  dontAutoPatchelf = true;
  dontPatchELF = true;
  dontStrip = true;
  configurePhase = ''
    export DENO_DIR=.deno; mkdir $DENO_DIR
    deno cache --unstable --lock=deno.lock --lock-write --reload main.ts
    mkdir -p $DENO_DIR/dl/release/v${pkgs.deno.version}
    ln -s ${compilier} $DENO_DIR/dl/release/v${pkgs.deno.version}/deno-${target}.zip
  '';
  buildPhase = ''
    mkdir -p $out/bin
    export OUTPUT=$out/bin/${pname}
    deno compile --unstable -A --lock=deno.lock --check --cached-only ./main.ts --output $OUTPUT --target=${target}
    TRAILER=$(tail -c 40 "$OUTPUT" | hexdump -v -e '1/1 "%02x "')
    autoPatchelf $OUTPUT
    echo "$TRAILER" | xxd -r -p >>"$OUTPUT"
  '';
};

[StarLederer]

One way to work around this would be to bundle the JS app and run it using Deno from Nix store, in fact this might even be a better solution altogether, unfortunately there are currently no feature-complete bundlers for Deno... deno bundle is deprecated and doesn't support NPM, deno_emit just doesn't work sometimes, esbuild and rollup are mentioned in the docs but neither have Deno plugins that resolve all kinds of dependencies Deno supports nor a bootstrap configuration for them.

I understand the drive to build a highly performant deno_emit but I think they should have built a solid Rollup plugin first to establish a good standard for feature completeness and compile-time customization and then focus on building a highly performant competitor from scratch

[CMCDragonkai]

@MrFoxPro you found the right trick. The autoPatchelf hook shouldn't be used that way though.

Instead inside my nix-shell with patchelf as a native build input:

deno compile --output y ./src/main.ts
TRAILER=$(tail -c 40 ./y | hexdump -v -e '1/1 "%02x "')
patchelf --set-interpreter $(cat $NIX_CC/nix-support/dynamic-linker) ./y
echo "$TRAILER" | xxd -r -p >>"./y"
./y
Hello World

Now works.

How did you find out what the last 40 bytes was?

[CMCDragonkai]

It appears the last 40 bytes is some sort of deno binary signature.

64 33 6e 30 6c 34 6e 64 00 00 00 00 04 d9 45 70 00 00 00 00 04 d9 47 15 00 00 00 00 04 d9 4a 95 00 00 00 00 04 d9 4a 99

Where there's an initial:

64 33 6e 30 6c 34 6e 64 00 00 00 00
d3 n0 l4 nd .... (followed by null bytes)

Leet speak d3nol4nd.

I wonder why this is missing in the deno compiled binary?

[06kellyjac]

I'm with @StarLederer on this

unfortunately there are currently no feature-complete bundlers for Deno

And some of that nix seems excessively complex at first glance but obviously I'd need to play with it (ideally with some sample code for ./api) to confirm

[CMCDragonkai]

Well in this case I believe this issue is focused on the compiled binary itself.

[CMCDragonkai]

@MrFoxPro how did you know that you needed exactly 40 bytes?

[mmastrac]

This should not be considered canonical information as this might change in the future, but the way we detect an eszip at the end of a file is like so:

1. Read the last 24 bytes from the file into three 8-byte words. Check the magic signature d3n0l4nd in the first word.
2. If it matches, read the eszip pos from the second pos and the metadata pos from the third word
3. Deserialize the metadata JSON from the metadata position in the file, the boot the eszip from the eszip from the eszip location in the file.

In theory you can use this to deconstruct and reconstruct a deno compiled binary.

[CMCDragonkai]

This should not be considered canonical information as this might change in the future, but the way we detect an eszip at the end of a file is like so:

1. Read the last 24 bytes from the file into three 8-byte words. Check the magic signature d3n0l4nd in the first word.
2. If it matches, read the eszip pos from the second pos and the metadata pos from the third word
3. Deserialize the metadata JSON from the metadata position in the file, the boot the eszip from the eszip from the eszip location in the file.

In theory you can use this to deconstruct and reconstruct a deno compiled binary.

I think this has already changed @mmastrac but from your explanation I did figure out what is going on.

The last 40 bytes is actually composed of 5x8 byte segments.

The first is a denoland signature taking up 8 bytes.

The next 4x8 byte sections are each byte positions (each are 32 bit numbers stored in a 64 bit section in big-endian).

1. Byte position of ESZIP archive - if you extract it, you can drop it in here https://eszip-viewer.deno.dev/ and it will work
2. Byte position of metadata JSON (there's no length data, but you just need to subtract this position to get only the eszip data)
3. Byte position of a null before the denoland
4. Byte position of the denoland signature

The fact that @MrFoxPro found that you had to re-append the 40 bytes at the end of the patchelfed binary initially seems to meant that the 40 bytes are gone, but this is not true, those 40 bytes still exist, it's just not at the end of the file anymore.

In fact, inspecting the patched binary, the patched binary has different ending bytes. I suspect patchelf by setting the interpreter path adds additional bytes to the end of the binary. (Normally it appears to add 4 KiB to the file length).

So I believe what is happening here is that deno's runtime tries to look up the end of the file to find the byte positions - in order to load the embedded JS file. However by patching the compiled binary, patchelf adds its own chunk of data at the end. Deno then cannot find this data anymore, and that's why the compiled binary no longer works.

By adding the 40 bytes to the end of the file, deno can find the signature and positions again. What's interesting is that, setting the interpreter path does not affect the expected byte positions, those positions are still preserved, and then deno can in fact find the ESZIP stuff and other stuff to make use of.

Now this is only by setting the interpreter path with patchelf, I'm not sure if patchelf may in fact affect the relative byte positions if it also modifies the rpath too.