Unhandled error event: Error: read ETIMEDOUT cause request stuck

[arifinoid]

Currently me and my friend works as a team on a project that use redis and websocket to handle download request from client. But we faced a problem on server side after we deploy it on production (live). We got unhandled error event: Error: read ETIMEDOUT when we tried to get jwt in session for authenticating stuff, and then we stuck on that state. Here is the SS and Sample code:

auth.js

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.initAuth = void 0;
const config_1 = __importDefault(require("../config"));
const cache_1 = __importDefault(require("./cache"));
// 12 hour expiration token
const TOKEN_EXPIRATION = 43200000;
// renew token on this last minutes
const EXPIRATION_WINDOW = 600000;
async function validateToken(decoded, request, callback) {
    try {
        const session = await cache_1.default.get(decoded.id); // <--stuck on this line
        if (session) {
            // if expiry time is less than 5 minutes update the cache to extend the session
            const now = new Date().getTime();
            if (now > session.exp) {
                // session expired
                cache_1.default.del(session.id);
                return { isValid: false };
            }
            if ((session.exp - now) <= EXPIRATION_WINDOW) {
                session.exp = now + TOKEN_EXPIRATION; // add another 10 minutes
                cache_1.default.set(session.id, session, TOKEN_EXPIRATION / 1000);
            }
            // session is valid
            return {
                isValid: true,
                credentials: session,
            };
        }
        // reply is null
        return { isValid: false };
    }
    catch (err) {
        console.log(err);
        return { isValid: false };
    }
}
function initAuth(server) {
    server.auth.strategy('jwt', 'jwt', {
        key: config_1.default.JWT_KEY,
        validate: validateToken,
        verifyOptions: { algorithms: ['HS256'] },
    });
    server.auth.default('jwt');
}
exports.initAuth = initAuth;

cache.js

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.kill = void 0;
const ioredis_1 = __importDefault(require("ioredis"));
const config_1 = __importDefault(require("../config"));
// redis config
const rc = {
    port: +config_1.default.REDIS_PORT,
    host: config_1.default.REDIS_HOST,
    auth_pass: config_1.default.REDIS_AUTH,
    useSsl: config_1.default.REDIS_USE_SSL,
    connectTimeout: 5000,
    retryStrategy: 5,
};
const CON = {}; // store redis connections as Object
function kill(type) {
    type = type || 'DEFAULT'; // kill specific connection or default one
    try {
        CON[type].end(false);
    }
    catch (e) {
        console.log(e);
    }
    delete CON[type];
}
exports.kill = kill;
function newConnection() {
    let redisCon;
    try {
        const options = { auth_pass: rc.auth_pass };
        if (rc.useSsl === 'true') {
            options.tls = { servername: rc.host };
        }
        redisCon = new ioredis_1.default(rc.port, rc.host, { password: rc.auth_pass });
    }
    catch (e) {
        console.log(e);
        throw e;
    }
    return redisCon;
}
function redisConnection(type) {
    type = type || 'DEFAULT'; // allow infinite types of connections
    if (!CON[type]) {
        CON[type] = newConnection();
    }
    return CON[type];
}
const cache = {
    set: async (key, value, exp) => {
        // for QA purpose disable dashboard cache
        if (key.startsWith('DASH:txbtffkshrvmfx1gmgxenua')) {
            return;
        }
        try {
            const client = redisConnection();
            let storedValue;
            if (typeof value === 'string') {
                storedValue = value;
            }
            else {
                storedValue = JSON.stringify(value);
            }
            if (exp !== undefined) {
                await client.set(key, storedValue, 'EX', exp);
            }
            else {
                await client.set(key, storedValue);
            }
        }
        catch (err) {
            console.log(err);
            throw err;
        }
    },
    get: async (key) => {
        try {
            console.log(1);
            const client = redisConnection();
            console.log(2);
            const reply = await client.get(key); // <--stuck on this line
            console.log(3); // <-- this line is not executed (unreachable)
            // reply might be null, so be sure to check it when calling this method
            if (reply) {
                try {
                    const o = JSON.parse(reply);
                    // Handle non-exception-throwing cases:
                    // Neither JSON.parse(false) or JSON.parse(1234) throw errors, hence the type-checking,
                    // but... JSON.parse(null) returns 'null', and typeof null === "object",
                    // so we must check for that, too.
                    if (o && typeof o === 'object' && o !== null) {
                        return o;
                    }
                }
                catch (e) {
                    console.log(e);
                }
            }
            // we don't have a valid json so return as is
            return reply;
        }
        catch (err) {
            console.log(err);
            throw err;
        }
    },
    del: async (key) => {
        try {
            const client = redisConnection();
            await client.del(key);
        }
        catch (err) {
            console.log(err);
            throw err;
        }
    },
};
exports.default = cache;

Any suggestions / tips for this situation ?? we are very grateful if there is any help from you. Thank you

[arifinoid]

<img src="https://i.ibb.co/qxG2GkF/Screen-Shot-2020-09-21-at-09-51-41.png" alt="postman-timeout"/>

<img src="https://i.ibb.co/2MPNW9j/Screen-Shot-2020-09-21-at-09-52-05.png" alt="code"/>

[lewchuk]

I am also running into this issue where a failure to connect just hangs requests. This has happened with all of the combinations of connection patterns I can think of:

• Connect right away.
• Use lazyConnect=true and call connect()
• Use lazyConnect=true and call a command like set.

I replicate this issue by simply not having redis running and the debug logs DEBUG=ioredis* produce:

> conn = new Redis({ port: 6379, host: 'localhost', lazyConnect:true });
> conn.connect().then(console.log).catch(console.error)
  ioredis:redis status[localhost:6379]: wait -> connecting +2m
  ioredis:redis status[127.0.0.1:6379]: connecting -> connect +2ms
  ioredis:redis write command[127.0.0.1:6379]: 0 -> info([]) +1ms
# Wait a long time
> conn.status
'connect'

I would definitely like to see the connection error propagate out to the caller, otherwise I am having to build my own connect timeout behaviour.

[lewchuk]

One poor workaround is to set enableOfflineQueue: false on the instance. In this case if you send any command while the connection isn't working they will immediately error:

Error: Stream isn't writeable and enableOfflineQueue options is false
    at Redis.sendCommand (/Users/stephenlewchuk/span/cloud/packages/app-api/node_modules/ioredis/built/redis/index.js:634:24)
    at Redis.set (/Users/stephenlewchuk/span/cloud/packages/app-api/node_modules/ioredis/built/commander.js:111:25)
    at repl:1:6

This does prevent the usage of lazy connect requiring you to manage connection on your own.