libjpeg.so.62: cannot open shared object file

[stevenspiel]

This is fantastic. Thank you!

I'm running into one issue while replicating what's going on in the tests locally:

index.js

let { execSync } = require('child_process');

exports.handler = async (event, context, callback) => {
  execSync("ldd /opt/bin/wkhtmltopdf", { encoding: "utf8", stdio: "inherit" });
  execSync("wkhtmltopdf -V", { encoding: "utf8", stdio: "inherit" });

  callback(null, {'foo': 'bar'});
};

$ sam build myFunction
...Build Succeeded...
$ sam local invoke -e event.json myFunction
Invoking index.handler (nodejs12.x)
arn:aws:lambda:us-east-1:347599033421:layer:wkhtmltopdf:1 is already cached. Skipping download
Requested to skip pulling images ...

Mounting /Users/stevenspiel/mobile/wkhtmltopdf-lambda/.aws-sam/build/myFunction as /var/task:ro,delegated inside runtime container
START RequestId: c361ad4f-866a-1450-c1dc-6a3e0f8318fe Version: $LATEST
    linux-vdso.so.1 (0x00007fff643f0000)
    libjpeg.so.62 => not found
    libpng15.so.15 => not found
    libXrender.so.1 => not found
    libfontconfig.so.1 => not found
    libfreetype.so.6 => not found
    libXext.so.6 => not found
    libX11.so.6 => not found
    libssl.so.10 => /lib64/libssl.so.10 (0x00007f7dc8912000)
    libcrypto.so.10 => /lib64/libcrypto.so.10 (0x00007f7dc84bd000)
    libz.so.1 => /lib64/libz.so.1 (0x00007f7dc82a8000)
    libdl.so.2 => /lib64/libdl.so.2 (0x00007f7dc80a4000)
    librt.so.1 => /lib64/librt.so.1 (0x00007f7dc7e9c000)
    libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f7dc7c7e000)
    libstdc++.so.6 => /lib64/libstdc++.so.6 (0x00007f7dc78fc000)
    libm.so.6 => /lib64/libm.so.6 (0x00007f7dc75bc000)
    libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007f7dc73a6000)
    libc.so.6 => /lib64/libc.so.6 (0x00007f7dc6ffb000)
    /lib64/ld-linux-x86-64.so.2 (0x00007f7dc8b81000)
    libgssapi_krb5.so.2 => /lib64/libgssapi_krb5.so.2 (0x00007f7dc6daf000)
    libkrb5.so.3 => /lib64/libkrb5.so.3 (0x00007f7dc6acb000)
    libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00007f7dc68c7000)
    libk5crypto.so.3 => /lib64/libk5crypto.so.3 (0x00007f7dc6696000)
    libkrb5support.so.0 => /lib64/libkrb5support.so.0 (0x00007f7dc6487000)
    libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007f7dc6283000)
    libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f7dc606d000)
    libselinux.so.1 => /lib64/libselinux.so.1 (0x00007f7dc5e46000)
    libpcre.so.1 => /lib64/libpcre.so.1 (0x00007f7dc5be2000)
wkhtmltopdf: error while loading shared libraries: libjpeg.so.62: cannot open shared object file: No such file or directory
2020-05-08T10:18:33.445Z    c361ad4f-866a-1450-c1dc-6a3e0f8318fe    ERROR   Invoke Error    {"errorType":"Error","errorMessage":"Command failed: wkhtmltopdf -V","status":127,"signal":null,"output":[null,null,null],"pid":31,"stdout":null,"stderr":null,"stack":["Error: Command failed: wkhtmltopdf -V","    at checkExecSyncError (child_process.js:630:11)","    at execSync (child_process.js:666:15)","    at Runtime.exports.handler (/var/task/index.js:21:3)","    at Runtime.handleOnce (/var/runtime/Runtime.js:66:25)"]}

It appears to be finding most of the libraries, but a few are missing. Do you know what might be the issue?

[brandonlim-hs]

Hi @stevenspiel,

I am unable to produce this using the AWS SAM NodeJS template.

Could you try removing the docker images for nodejs12.x from samcli/lambda and lambci/lambda then run sam local invoke -e event.json myFunction again? Otherwise, are you be able to share a sample project that can reproduce this?

[stevenspiel]

@brandonlim-hs That did it! Thank you!

[stevenspiel]

Ok, so it worked once right after I removed the docker images, but all subsequent attempts have returned the original error.

$ docker rmi $(docker images -a -q)
$ docker images -a
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
$ sam build myFunction
...Build Succeeded...
$ sam local invoke -e event.json myFunction
Invoking index.handler (nodejs12.x)
arn:aws:lambda:us-east-1:347599033421:layer:wkhtmltopdf:1 is already cached. Skipping download
Image was not found.
Building image...
Requested to skip pulling images ...

Mounting /Users/stevenspiel/mobile/myFunction.aws-sam/build/animalReport as /var/task:ro,delegated inside runtime container
START RequestId: 80d477de-e87b-18ab-ae6b-d6be7a9543f8 Version: $LATEST
    linux-vdso.so.1 (0x00007ffc607f8000)
    libjpeg.so.62 => not found
    libpng15.so.15 => not found
    libXrender.so.1 => not found
    libfontconfig.so.1 => not found
    libfreetype.so.6 => not found
    libXext.so.6 => not found
    libX11.so.6 => not found
    libssl.so.10 => /lib64/libssl.so.10 (0x00007f5a3819f000)
    libcrypto.so.10 => /lib64/libcrypto.so.10 (0x00007f5a37d4a000)
    libz.so.1 => /lib64/libz.so.1 (0x00007f5a37b35000)
    libdl.so.2 => /lib64/libdl.so.2 (0x00007f5a37931000)
    librt.so.1 => /lib64/librt.so.1 (0x00007f5a37729000)
    libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f5a3750b000)
    libstdc++.so.6 => /lib64/libstdc++.so.6 (0x00007f5a37189000)
    libm.so.6 => /lib64/libm.so.6 (0x00007f5a36e49000)
    libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007f5a36c33000)
    libc.so.6 => /lib64/libc.so.6 (0x00007f5a36888000)
    /lib64/ld-linux-x86-64.so.2 (0x00007f5a3840e000)
    libgssapi_krb5.so.2 => /lib64/libgssapi_krb5.so.2 (0x00007f5a3663c000)
    libkrb5.so.3 => /lib64/libkrb5.so.3 (0x00007f5a36358000)
    libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00007f5a36154000)
    libk5crypto.so.3 => /lib64/libk5crypto.so.3 (0x00007f5a35f23000)
    libkrb5support.so.0 => /lib64/libkrb5support.so.0 (0x00007f5a35d14000)
    libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007f5a35b10000)
    libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f5a358fa000)
    libselinux.so.1 => /lib64/libselinux.so.1 (0x00007f5a356d3000)
    libpcre.so.1 => /lib64/libpcre.so.1 (0x00007f5a3546f000)
wkhtmltopdf: error while loading shared libraries: libjpeg.so.62: cannot open shared object file: No such file or directory
2020-05-08T16:33:23.098Z    80d477de-e87b-18ab-ae6b-d6be7a9543f8    ERROR   Invoke Error    {"errorType":"Error","errorMessage":"Command failed: wkhtmltopdf -V","status":127,"signal":null,"output":[null,null,null],"pid":28,"stdout":null,"stderr":null,"stack":["Error: Command failed: wkhtmltopdf -V","    at checkExecSyncError (child_process.js:630:11)","    at execSync (child_process.js:666:15)","    at Runtime.exports.handler (/var/task/index.js:23:3)","    at Runtime.handleOnce (/var/runtime/Runtime.js:66:25)"]}
END RequestId: 80d477de-e87b-18ab-ae6b-d6be7a9543f8
REPORT RequestId: 80d477de-e87b-18ab-ae6b-d6be7a9543f8  Init Duration: 1141.15 ms   Duration: 28.33 ms  Billed Duration: 100 ms Memory Size: 256 MB Max Memory Used: 57 MB

{"errorType":"Error","errorMessage":"Command failed: wkhtmltopdf -V"}

$ docker images -a
REPOSITORY          TAG                                    IMAGE ID            CREATED             SIZE
samcli/lambda       nodejs12.x-13b47fd036d464fd1322449e9   9a469e9410ff        2 minutes ago       471MB
lambci/lambda       nodejs12.x                             f71293a4a833        38 hours ago        380MB

I'm not sure why lambci/lambda says it was created 38 hours ago, when it was just deleted and recreated. I'm also unsure why it would work once and not again. Any thoughts/ideas would be really appreciated!

[stevenspiel]

Ok, I found the original issue. When dependencies were directly inside the lambda function, I had to use a workaround to include them

process.env['PATH'] = process.env['PATH'] + ':' + process.env['LAMBDA_TASK_ROOT'];
process.env['PATH'] = process.env['PATH'] + ':' + process.env['LAMBDA_TASK_ROOT'] + '/bin';
process.env['LD_LIBRARY_PATH'] = process.env['LAMBDA_TASK_ROOT'] + '/bin';

When that is present, some of the libraries could no longer be found and required deleting the images and starting over. Once I removed those lines, everything was good 👍

[brandonlim-hs]

Glad that you've got it sorted! 👍