agebrock / tunnel-ssh

Easy ssh tunneling
MIT License
356 stars 96 forks source link

Application crashing when `No Response from server` #113

Open nick22985 opened 1 year ago

nick22985 commented 1 year ago

Hey, I am trying to handle all errors to stop this from crashing my application if the server goes down. Instead it will gracefully handle them and reconnect.

Below is what I currently have but I am facing a issue with if the server goes down on the server it will crash my application. I have purposly stopped mongo on the server that it is connecting to simulate this.

  bot:info ----------------------Starting bot---------------------- +0ms
  bot:sshTunnel SSH Tunnel found in .env file, creating tunnel... +0ms
try connection
  bot:sshTunnel SSH Tunnel created! +0ms
  bot:mongo Connecting to the database... +0ms
  bot:mongoosy:connecting Connecting to MongoDB.... +0ms
  bot:sshTunnel SSH Tunnel Server Connected +0ms
  bot:sshTunnel:warn SSH Tunnel Conn Error (handled) Error: (SSH) Channel open failure: Connection refused
    at onChannelOpenFailure (C:\bot\node_modules\ssh2\lib\utils.js:16:11)
    at CHANNEL_OPEN_FAILURE (C:\bot\node_modules\ssh2\lib\client.js:572:11)
    at 92 (C:\bot\node_modules\ssh2\lib\protocol\handlers.misc.js:881:16)
    at Protocol.onPayload (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:2052:10)
    at AESGCMDecipherNative.decrypt (C:\bot\node_modules\ssh2\lib\protocol\crypto.js:987:26)
    at Protocol.parsePacket [as _parse] (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:2021:25)
    at Protocol.parse (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:306:16)
    at Socket.<anonymous> (C:\bot\node_modules\ssh2\lib\client.js:775:21)
    at Socket.emit (node:events:511:28)
    at Socket.emit (node:domain:489:12) {
  reason: 2
} +0ms
  bot:sshTunnel SSH Tunnel found in .env file, creating tunnel... +0ms
try connection
  bot:sshTunnel:warn SSH Tunnel Conn Closed +0ms
  bot:sshTunnel SSH Tunnel created! +0ms
  bot:sshTunnel SSH Tunnel Server Connected +0ms
  bot:sshTunnel SSH Tunnel Server Connected +0ms
  bot:sshTunnel:warn SSH Tunnel Conn Error (handled) Error: (SSH) Channel open failure: Connection refused
    at onChannelOpenFailure (C:\bot\node_modules\ssh2\lib\utils.js:16:11)
    at CHANNEL_OPEN_FAILURE (C:\bot\node_modules\ssh2\lib\client.js:572:11)
    at 92 (C:\bot\node_modules\ssh2\lib\protocol\handlers.misc.js:881:16)
    at Protocol.onPayload (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:2052:10)
    at AESGCMDecipherNative.decrypt (C:\bot\node_modules\ssh2\lib\protocol\crypto.js:987:26)
    at Protocol.parsePacket [as _parse] (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:2021:25)
    at Protocol.parse (C:\bot\node_modules\ssh2\lib\protocol\Protocol.js:306:16)
    at Socket.<anonymous> (C:\bot\node_modules\ssh2\lib\client.js:775:21)
    at Socket.emit (node:events:511:28)
    at Socket.emit (node:domain:489:12) {
  reason: 2
} +0ms
  bot:sshTunnel SSH Tunnel found in .env file, creating tunnel... +0ms
try connection
  bot:sshTunnel:warn SSH Tunnel Conn Closed +0ms
C:\bot\node_modules\ssh2\lib\client.js:826
      const err = new Error('No response from server');
                  ^
Error: No response from server
    at Socket.<anonymous> (C:\bot\node_modules\ssh2\lib\client.js:826:19)
    at Socket.emit (node:events:511:28)
    at Socket.emit (node:domain:489:12)
    at TCP.<anonymous> (node:net:335:12)
[nodemon] app crashed - waiting for file changes before starting...
async setupTunnel() {
        if (process.env.SSH_TUNNEL_HOST) {
            this.$debug.extend('sshTunnel')('SSH Tunnel found in .env file, creating tunnel...');
            if (!process.env.SSH_TUNNEL_HOST) throw new Error('SSH_TUNNEL_HOST not found in .env file');
            if (!process.env.SSH_TUNNEL_PORT) throw new Error('SSH_TUNNEL_PORT not found in .env file');
            if (!process.env.SSH_TUNNEL_SSH_PORT) throw new Error('SSH_TUNNEL_SSH_PORT not found in .env file');
            if (!process.env.SSH_TUNNEL_USERNAME) throw new Error('SSH_TUNNEL_USERNAME not found in .env file');
            if (!process.env.SSH_TUNNEL_PRIVATEKEY) throw new Error('SSH_TUNNEL_PRIVATEKEY not found in .env file');
            if (!process.env.SSH_TUNNEL_SRC_ADDR) throw new Error('SSH_TUNNEL_SRC_ADDR not found in .env file');
            if (!process.env.SSH_TUNNEL_DST_ADDR) throw new Error('SSH_TUNNEL_DST_ADDR not found in .env file');
            const tunnelOptions = {
                autoClose: false,
            };
            const serverOptions = {
                port: parseInt(process.env.SSH_TUNNEL_PORT),
            };
            const sshOptions = {
                host: process.env.SSH_TUNNEL_HOST,
                port: process.env.SSH_TUNNEL_SSH_PORT,
                username: process.env.SSH_TUNNEL_USERNAME,
                privateKey: require('fs').readFileSync(process.env.SSH_TUNNEL_PRIVATEKEY),
                passphrase: process.env.SSH_TUNNEL_PASSPHRASE,
                keepaliveInterval: 10000,
                readyTimeout: 30000,
                keepaliveCountMax: 10,
                debug: (msg) => {
                    this.$debug.extend('sshTunnel:debug')(msg);
                },
            };
            const forwardOptions = {
                srcAddr: process.env.SSH_TUNNEL_SRC_ADDR,
                srcPort: parseInt(process.env.SSH_TUNNEL_PORT),
                dstAddr: process.env.SSH_TUNNEL_DST_ADDR,
                dstPort: parseInt(process.env.SSH_TUNNEL_PORT),
            };
            try {
                console.log('try connection');
                this.sshTunnel = await createTunnel(tunnelOptions, serverOptions, sshOptions, forwardOptions)
                    .then((sshTunnel) => {
                        let [server, client] = sshTunnel;
                        if (!server || !client) return this.$debug.extend('sshTunnel:error')('SSH Tunnel', 'server or client not found');
                        client.on('error', (err) => {
                            Promise.resolve()
                                .then(() => (server ? server.close() : null))
                                .then(() => (client ? client.end() : null))
                                .then(() => this.setupTunnel());
                            this.$debug.extend('sshTunnel:warn')('SSH Tunnel Conn Error (handled)', err);
                        });

                        client.on('close', () => {
                            if (server) server.close();
                            this.$debug.extend('sshTunnel:warn')('SSH Tunnel Conn Closed');
                        });

                        server.on('close', () => {
                            this.$debug.extend('sshTunnel:warn')('SSH Tunnel Server', 'closed');
                            client.end();
                        });

                        server.on('error', (err) => {
                            Promise.resolve()
                                .then(() => (server ? server.close() : null))
                                .then(() => (server ? client.end() : null))
                                .then(() => this.setupTunnel());
                            this.$debug.extend('sshTunnel:warn')('SSH Tunnel Server Error (Handled)', err);
                        });
                        server.on('connection', (connection) => {
                            connection.on('error', (err) => {
                                this.$debug.extend('sshTunnel:warn')('SSH server connection error:', err);
                            });
                            connection.on('close', () => {
                                this.$debug.extend('sshTunnel:warn')('SSH server connection closed');
                            });
                            connection.on('end', () => {
                                this.$debug.extend('sshTunnel:warn')('SSH server connection ended');
                            });
                            this.$debug.extend('sshTunnel')('SSH Tunnel Server Connected');
                        });
                        return sshTunnel;
                    })
                    .catch((err) => {
                        this.$debug.extend('sshTunnel:error')('SSH Tunnel', err);
                    });
            } catch (e) {
                this.$debug.extend('sshTunnel:error')('SSH Tunnel', e);
            }
            this.$debug.extend('sshTunnel')('SSH Tunnel created!');
        } else this.$debug.extend('sshTunnel')('No SSH Tunnel found in .env file');
nick22985 commented 1 year ago

Also with the new update it is possible for the ssConnection.isBroken to be undefined and causes the package to fall over

            if (sshConnection.isBroken) {
                              ^
TypeError: Cannot read properties of undefined (reading 'isBroken')
agebrock commented 1 year ago

Hi there, you did some impressive job here :-)

I will do some additional testing on that topic, and I am planning to release a pre-release version , that could make your life much easier.

Thanks for the effort I will come back to you !

negativems commented 1 year ago

Also with the new update it is possible for the ssConnection.isBroken to be undefined and causes the package to fall over

            if (sshConnection.isBroken) {
                              ^
TypeError: Cannot read properties of undefined (reading 'isBroken')

I had this error multiple times

nick22985 commented 9 months ago

I ended up rewriting this to suit my needs. I did not do all the auto connection close things but this may help with solving the issues that I was facing here.

import { Client, type ConnectConfig } from 'ssh2';
import HandlerClient from './handlers/client.js';
import net, { type ListenOptions, type Server } from 'net';
import { type Debugger } from 'debug';
export interface ITunnelOptions {
    autoClose: boolean;
    reconnectOnError: boolean;
}

export interface IForwardOptions {
    srcAddr: string;
    srcPort: number;
    dstAddr: string;
    dstPort: number;
}

export default class sshTunnel {
    discordClient: HandlerClient;
    debug: Debugger;
    tunnelOptions: ITunnelOptions;
    listenOptions: ListenOptions;
    connectConfig: ConnectConfig;
    forwardOptions: IForwardOptions;

    // Client, Server
    server: Server | undefined;
    client: Client | undefined;

    clientReconnectInterval: NodeJS.Timeout | undefined;

    constructor(options: { discordClient: HandlerClient; tunnelOptions: ITunnelOptions; listenOptions: ListenOptions; connectConfig: ConnectConfig; forwardOptions: IForwardOptions }) {
        this.discordClient = options.discordClient;
        this.tunnelOptions = Object.assign({ autoClose: false, reconnectOnError: false }, options.tunnelOptions || {});
        this.listenOptions = options.listenOptions;
        this.connectConfig = Object.assign({ port: 22, username: 'root' }, options.connectConfig);
        this.forwardOptions = Object.assign({ dstAddr: '0.0.0.0' }, options.forwardOptions);

        this.debug = this.discordClient.$debug.extend('sshTunnel');
        this.debug('Starting SSH Tunnel');
    }

    createServer() {
        const $debug = this.debug.extend('createServer');
        $debug('Creating Server');
        return Promise.resolve()
            .then(() => net.createServer())
            .then((server) => {
                return new Promise((resolve, reject) => {
                    let errorHandler = (err) => {
                        $debug('Error', err);
                        reject(err);
                    };
                    server.on('error', errorHandler);
                    process.on('uncaughtException', errorHandler);

                    server.on('close', () => {
                        $debug('Server Close');
                        if (this.tunnelOptions.reconnectOnError) {
                            this.createServer().then(() => this.serverEventListeners());
                        }
                    });

                    server.on('drop', () => {
                        $debug('Server Drop');
                    });

                    server.listen(this.listenOptions);

                    server.on('listening', () => {
                        process.removeListener('uncaughtException', errorHandler);
                        $debug('Server Listening');
                        this.server = server;
                        resolve(server);
                    });
                });
            });
    }

    createSSHClient() {
        const $debug = this.debug.extend('createSSHClient');
        $debug('Creating SSH Client');
        // make sure client is closed
        if (this.client) {
            $debug('Client has old connection and is trying to reconnect killing old client');
            this.client.end();
            this.client = undefined;
        }
        return new Promise((resolve, reject) => {
            let conn: Client = new Client();
            conn.on('ready', () => {
                this.client = conn;
                resolve(conn);
            });
            conn.on('error', reject);

            conn.on('close', () => {
                $debug('Client Close');
                if (this.tunnelOptions.reconnectOnError) {
                    setTimeout(() => {
                        return this.createSSHClient()
                            .then(() => this.clientEventListeners())
                            .then(() => $debug('reconnected to client'))
                            .catch(() => {
                                $debug.extend('error')('failed to reconnect to client');
                            });
                    }, 10000);
                }
            });

            conn.on('end', () => {
                $debug('Client End');
            });
            Promise.resolve()
                .then(() => conn.connect(this.connectConfig))
                .catch((err: Error) => {
                    $debug.extend('error')('failed to connect', err);
                    throw err;
                });
        }).catch((err: Error) => {
            $debug.extend('error')('failed to createSSHClient', err);
            throw err;
        });
    }

    serverEventListeners() {
        const $debug = this.debug.extend('serverEventListeners');
        $debug('Creating Server Event Listeners');
        if (!this.server) throw new Error('No Server found');
        // reco logic
        if (this.tunnelOptions.reconnectOnError)
            this.server.on('error', (err) => {
                $debug.extend('error')('Server Error', err);
                return Promise.resolve().then(() => this.createServer().then(() => this.serverEventListeners()));
            });

        this.server.on('connection', (connection) => {
            $debug('Server Connection', connection.address());
            if (!this.client) {
                $debug('No Client Connection');
                return connection.end(); // Kick the connection trying to connect to the server
            }
            connection.on('error', (err) => {
                $debug('Connection Closed error', err);
            });
            // This is the mongo Connection itself
            return this.client.forwardOut(this.forwardOptions.srcAddr, this.forwardOptions.srcPort, this.forwardOptions.dstAddr, this.forwardOptions.dstPort, (err, stream) => {
                if (err) {
                    $debug.extend('error')('Server Connection Error', connection.address());
                    $debug('server con err', err);

                    setTimeout(() => {
                        return connection.end(); // end user connection to server
                    }, 10000);
                } else
                    connection
                        .pipe(stream)
                        .pipe(connection)
                        .on('close', (test) => {
                            $debug('Server Connection Close', test);
                        })
                        .on('error', (err: Error) => {
                            $debug('Connection closed from server (Usually this is from the client not closing the connection) (usually ignore this)', err);
                        });
            });
        });
        this.server.on('close', () => {
            $debug('Server Close');
            if (this.client) this.client.end();
        });
    }

    clientEventListeners() {
        const $debug = this.debug.extend('clientEventListeners');
        $debug('Creating Client Event Listeners');
        if (!this.client) throw new Error('No Client found');

        this.client.on('ready', () => {
            $debug('Client Ready');
        });
        this.client.on('close', () => {
            $debug('Client Close');
            // if (this.server) this.server.close();
        });
    }

    createTunnel() {
        this.debug('Creating SSH Tunnel');
        return Promise.resolve()
            .then(() => this.createSSHClient()) // Create SSH Client
            .then(() => this.createServer()) // Create Server
            .then(() => this.serverEventListeners()) // server event listners
            .then(() => this.debug('SSH Tunnel Created'));
    }
}
ghusse commented 6 months ago

I experience the same issue in my tests (and possibly in the app I'm working on):

        it('should throw an error when the destination url is incorrect', async () => {
          const privateKey = await readFile(path.resolve(__dirname, 'ssh-config', 'id_rsa'));
          await expect(
            buildMongooseInstance({
              uri: 'mongodb://username:secret@invalid_host:27017/movies?authSource=admin',
              connection: {
                ssh: {
                  host: '127.0.0.1',
                  port: 2224,
                  username: 'forest',
                  privateKey,
                },
                socketTimeoutMS: 10,
                connectTimeoutMS: 10,
                serverSelectionTimeoutMS: 10,
              },
            }),
          ).rejects.toThrow(
            new ConnectionError(
              'mongodb://forest:***@invalid_host:27017/movies?authSource=admin',
              'Server selection timed out after 10 ms',
            ),
          );
        });

        it('should pass', async () => {
          await new Promise(resolve => {
            setTimeout(resolve, 1000);
          });
          expect(true).toBe(true);
        });

The first test checks that the SSH error is correctly handled by our code when the URL is incorrect.

BUT when running this test suite, the second test actually fails!

I think this is caused by this line: https://github.com/agebrock/tunnel-ssh/blob/master/index.js#L123

sshConnection.forwardOut(
                forwardOptionsLocal.srcAddr,
                forwardOptionsLocal.srcPort,
                forwardOptionsLocal.dstAddr,
                forwardOptionsLocal.dstPort, (err, stream) => {
                    if (err) {
                        if (server) {
                            server.close()
                        }
                        throw err;
                    } else {
                        clientConnection.pipe(stream).pipe(clientConnection);
                    }
                });

We should not throw the error in this callback, but instead send it back to sshConnection