centrifugal / centrifugo

Scalable real-time messaging server in a language-agnostic way. Self-hosted alternative to Pubnub, Pusher, Ably. Set up once and forever.
https://centrifugal.dev
Apache License 2.0
8.35k stars 593 forks source link

[question] When I want to test it up to 5k no problem, But then mistakes happen ? #901

Closed rawezhcode closed 19 hours ago

rawezhcode commented 5 days ago

image

benchmark.js File

import ws from "k6/ws";
import { check } from "k6";
import crypto from "k6/crypto";
import encoding from "k6/encoding";

const hmacSecret = "1d2a52dc-9f69-4f0a-a5b8-xxxxxxxxxxx";

const algToHash = {
  HS256: "sha256",
  HS384: "sha384",
  HS512: "sha512",
};

function sign(data, hashAlg, secret) {
  let hasher = crypto.createHMAC(hashAlg, secret);
  hasher.update(data);
  // Some manual base64 encoding as `Hasher.digest(encodingType)` doesn't support that encoding type yet.
  return hasher
    .digest("base64")
    .replace(/\//g, "_")
    .replace(/\+/g, "-")
    .replace(/=/g, "");
}

function encode(payload, secret, algorithm) {
  algorithm = algorithm || "HS256";
  let header = encoding.b64encode(
    JSON.stringify({ typ: "JWT", alg: algorithm }),
    "rawurl"
  );
  payload = encoding.b64encode(JSON.stringify(payload), "rawurl");
  let sig = sign(header + "." + payload, algToHash[algorithm], secret);
  return [header, payload, sig].join(".");
}

// Generate a token for each user separately using HMAC JWT
function generateToken(userId) {
  const payload = {
    sub: userId,
    exp: Math.floor(Date.now() / 1000) + 60, // Expires in 60 seconds
  };
  return encode(payload, hmacSecret, "HS256");
}

export let options = {
  stages: [
    { duration: "120s", target: 15000 },
    { duration: "300s", target: 15000 },
  ],
};

export default function () {
  const url = "wss://xxxxxxx-xxxx-xxx/connection/websocket";

  const user = `user_${__VU}_${__ITER}`;
  const token = generateToken(user);

  const response = ws.connect(url, {}, function (socket) {
    socket.on("open", () => {
      const connectCommand = JSON.stringify({
        id: 1,
        connect: { token: token },
      });
      const subscribeCommand = JSON.stringify({
        id: 2,
        subscribe: { channel: "exams:qa" },
      });
      socket.send(connectCommand + "\n" + subscribeCommand);
    });

    socket.on("message", (message) => {
      // Respond to server pings.
      const substrings = message.split("\n");
      if (substrings.includes("{}")) {
        socket.send("{}");
      }
    });

    socket.on("close", () => {});

    socket.on("error", (error) => {
      console.log("WebSocket error: ", error);
    });

    socket.setTimeout(() => {
      socket.close();
    }, 60000);
  });

  check(response, { "status is 101": (r) => r && r.status === 101 });
}

config.json

{
  "admin": true,
  "token_hmac_secret_key": "xxxx-xxxx-xxxx",
  "admin_password": "xxxx-xxxx-xxxx",
  "admin_secret": "xxxx-xxxx-xxxx",
  "api_key": "xxxx-xxxx-xxxx",
  "allowed_origins": ["*"],
  "allow_subscribe_for_client": true,
  "allow_presence_for_subscriber": true,
  "allow_presence_for_client": true,
  "allow_subscribe_for_anonymous": true,
  "allow_anonymous_connect_without_token": true,

  "namespaces": [{
    "name": "presence-exams",
    "presence": true,
    "allow_subscribe_for_anonymous": true,
    "allow_publish_for_subscriber": true,
    "allow_publish_for_anonymous": true,
    "allow_presence_for_subscriber": true,
    "allow_presence_for_anonymous":true,
    "allow_subscribe_for_client": true
  }],
   "tls": {
     "enabled": true,
     "cert_pem_file": "/var/www/httpd-cert/xxxx-xxxx-xxxx-10-17-01-11_49.crt",
     "key_pem_file": "/var/www/httpd-cert/xxxx-xxxx-xxxx-10-17-01-11_49.key"
  }

}

centridugo.service

[Unit]
Description=Centrifugo Websocket Server
After=network.target syslog.target redis-server.service

[Service]
LimitNOFILE=30000
WorkingDirectory=/var/www/xxxx-xxxx-xxxx/data/www/xxxx-xxxx-xxxx
ExecStartPre=sudo ./centrifugo checkconfig --config config.json
ExecStart=sudo ./centrifugo --config=config.json
ExecReload=/bin/kill -HUP $MAINPID
ExecStop=/bin/kill -SIGTERM $MAINPID
TimeoutStopSec=5
KillMode=control-group
RestartSec=3
SyslogIdentifier=/var/www/xxxx-xxxx/data/www/xxxx-xxxx-xxxx/centrifugo.log
StandardOutput=append:/var/www/xxxx-xxxx/data/www/xxxx-xxxx-xxxx/centrifugo.log
StandardError=append:/var/www/xxxx-xxxx/data/www/xxxx-xxxx-xxxx/centrifugo.log

[Install]
WantedBy=multi-user.target
Alias=centrifugo.service

my VPS:

4 core 10Gb Ram 100Gb nvme

FZambia commented 2 days ago

Hello @rawezhcode

Can be caused by k6 generator - it may be too slow, and does not processing messages fast enough. In general, no hard limit on Centrifugo side which can cause such behaviour. Pay attention to resource usage on generator and server side. We have customers with 100k, 50k clients per one Centrifugo node.

rawezhcode commented 2 days ago

After some testing, I used the K6, and the problem was with the K6, not the Centrifugo.

But theoretically, how much can this VPS withstand?

my VPS:

4 core 10Gb Ram 100Gb nvme

FZambia commented 1 day ago

It is always hard to say.. For 10GB RAM the main limiting factor will be CPU most probably (if history is not used, though need to also estimate presence overhead here), the usage depends on load profile, so it's hard for me to estimate. Load testing should give answers, it's a bit difficult to make for WebSocket properly unfortunately as you can see. I believe that sth like 100k connections should be possible on Linux if we consider a workload of abstract messenger application. I am making assumptions on the numbers based on real-life numbers achieved in the real experiment - https://centrifugal.dev/blog/2020/02/10/million-connections-with-centrifugo