search

centrifugal / centrifugo

Scalable real-time messaging server in a language-agnostic way. Self-hosted alternative to Pubnub, Pusher, Ably. Set up once and forever.
https://centrifugal.dev
Apache License 2.0
8.44k stars 598 forks source link

[bug] Centrifugo Panic with Redis Cluster #862

Closed boskiv closed 4 months ago

boskiv commented 4 months ago

Describe the bug. A clear and concise description of what the bug is.

centrifugo panic

centrifugo-85d947bbdb-pznbx centrifugo {"level":"error","channel":"public:presence","client":"d21e6be1-fd01-44e5-bd00-30bb8d5e34bc","user":"","time":"2024-07-17T12:15:32Z","message":"timeout waiting for subscribe to finish"}
centrifugo-85d947bbdb-pznbx centrifugo panic: multi key command with different key slots are not allowed
centrifugo-85d947bbdb-pznbx centrifugo
centrifugo-85d947bbdb-pznbx centrifugo goroutine 251 [running]:
centrifugo-85d947bbdb-pznbx centrifugo github.com/redis/rueidis/internal/cmds.check(...)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/redis/rueidis@v1.0.41/internal/cmds/cmds.go:368
centrifugo-85d947bbdb-pznbx centrifugo github.com/redis/rueidis/internal/cmds.EvalshaNumkeys.Key({0xc0020229a0?, 0xfd8?, 0x210?}, {0xc002070ec0, 0x4, 0x0?})
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/redis/rueidis@v1.0.41/internal/cmds/gen_scripting.go:256 +0x1c5
centrifugo-85d947bbdb-pznbx centrifugo github.com/redis/rueidis.(*Lua).Exec(0xc000cfe3f0, {0x20f1f48, 0x2ebd4a0}, {0x20ff0e0, 0xc000b0c000}, {0xc002070ec0, 0x4, 0x4}, {0xc001ec3e00, 0x6, ...})
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/redis/rueidis@v1.0.41/lua.go:41 +0x5c7
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*RedisPresenceManager).addPresence(0xc000575d00, 0xc0005ffe00, {0xc00200ddb0?, 0xe07333?}, {0xc00096a5a0?, 0x7f7373496108?}, 0x50?)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/presence_redis.go:152 +0xa5
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*RedisPresenceManager).AddPresence(0xc0001a5508?, {0xc00200ddb0?, 0xc001f3c960?}, {0xc00096a5a0?, 0xc002078140?}, 0xe12ec0?)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/presence_redis.go:116 +0xb2
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*Node).addPresence(0xc000435688, {0xc00200ddb0, 0xf}, {0xc00096a5a0, 0x24}, 0xc001f3c960)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/node.go:1169 +0x8e
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*Client).subscribeCmd(_, _, {{0x0, {0x0, 0x0, 0x0}, 0x1, 0x0, 0x0, 0x0, ...}, ...}, ...)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/client.go:2810 +0x6d8
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*Client).handleSubscribe.func1({{0x0, {0x0, 0x0, 0x0}, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}, ...)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/client.go:1652 +0x1e5
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifugo/v5/internal/client.(*Handler).Setup.func3.3.1()
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/work/centrifugo/centrifugo/internal/client/handler.go:215 +0xdf
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifugo/v5/internal/client.(*Handler).runConcurrentlyIfNeeded(0x0?, {0x20f26a0?, 0xc0001de420?}, 0x7f7373496108?, 0x40?, 0xc000100808?)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/work/centrifugo/centrifugo/internal/client/handler.go:301 +0xfb
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifugo/v5/internal/client.(*Handler).Setup.func3.3({{0xc00200ddb0, 0xf}, {0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0, 0x0}, ...)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/work/centrifugo/centrifugo/internal/client/handler.go:213 +0x16f
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*Client).handleSubscribe(0xc002078140, 0xc000496320, 0xc000152e10, {0x1605160?, 0x3000000000000001?, 0x2e4f540?}, 0x0)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/client.go:1669 +0x282
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*Client).dispatchCommand(0xc002078140, 0xc000152e10, 0x33)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/client.go:1241 +0x6a5
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*Client).HandleCommand(0xc002078140, 0xc000152e10, 0x33)
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/client.go:1081 +0xc5
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.HandleReadFrame(0xc002078140, {0x7f732c6dd868, 0xc000150140})
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/handler_websocket.go:255 +0x138
centrifugo-85d947bbdb-pznbx centrifugo github.com/centrifugal/centrifuge.(*WebsocketHandler).ServeHTTP.func2()
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/handler_websocket.go:221 +0x637
centrifugo-85d947bbdb-pznbx centrifugo created by github.com/centrifugal/centrifuge.(*WebsocketHandler).ServeHTTP in goroutine 249
centrifugo-85d947bbdb-pznbx centrifugo  /home/runner/go/pkg/mod/github.com/centrifugal/centrifuge@v0.32.3-0.20240714053108-49aa7830216c/handler_websocket.go:180 +0x62c

Versions

Centrifugo version is 5.4.2 from helm centrifugo-11.8.2 Client library used is centrifuge-<???> of version <???> Operating system is k8s

Steps to Reproduce How can the bug be triggered? Install binami/redis-cluster Install centrifugo from helm

helm values

replicaCount: 3

metrics:
  enabled: true
  serviceMonitor:
    enabled: true
secrets:
  redisPassword: "xxxxxx"

config:
  engine: "redis"
  redis_address: "redis://redis-redis-cluster.redis:6379"
  allow_subscribe_for_client: true
  allow_publish_for_client: true
  allow_publish_for_subscriber: true
  token_hmac_secret_key: xxxxxxx-xxxxx-xxxxxx-xxxxxxx
  allow_anonymous_connect_without_token: true
  allow_subscribe_for_anonymous: true
  websocket_message_size_limit: 1048576
  admin_password: xxxxxxx-xxxxx-xxxxxx-xxxxxxx
  admin_secret: xxxxxxx-xxxxx-xxxxxx-xxxxxxx
  api_key: xxxxxxx-xxxxx-xxxxxx-xxxxxxx
  allowed_origins: ["*"]
  namespaces:
    - name: public 
      presence: true
      allow_subscribe_for_client: true
      allow_subscribe_for_anonymous: true
      allow_presence_for_client: true
      allow_presence_for_anonymous: true 

ingress:
  enabled: true
  ingressClassName: "nginx"
  pathType: Prefix
  labels: {}
  annotations:
    external-dns.alpha.kubernetes.io/hostname: ws.k8s.local
  hosts:
    - host: ws.k8s.local
      paths:
        - /
  tls: 
    - secretName: ws.k8s.local-tls
      hosts:
        - ws.k8s.local

Expected behavior What output or behaviour were you expecting instead?

No panic or restrict usage with cluster solution

Code Snippets A minimum viable code snippet can be useful.

React App


import { useCentrifugo } from '../services/centrifugo.service';

export function App() {

  const centrifuge = useCentrifugo();

  centrifuge?.getSubscription('chat')?.publish({ input: 'Hello' });

  let subscription = centrifuge?.getSubscription('public:presence');
  if (!subscription) {
    subscription = centrifuge?.newSubscription('public:presence');
  }
  subscription?.subscribe();
  subscription?.presenceStats().then((presence) => {
    console.log(presence.numClients);
  });

  return (
    <div>
    Hi
    </div>
  );
}

export default App;

Centrifugo hook

import { Centrifuge } from "centrifuge";
import { useEffect, useState } from "react";

export const WS_ENDPOINT = 'ws://localhost:8000/connection/websocket';

export function useCentrifugo() {
  const [centrifuge, setCentrifuge] = useState<Centrifuge | null>(null);

  useEffect(() => {
    const init = async () => {
      const centrifuge = new Centrifuge(WS_ENDPOINT, {
        debug: true
      });
      centrifuge.connect();
      setCentrifuge(centrifuge);
    };

    init();
  }, []);

  return centrifuge;
}
FZambia commented 4 months ago

Hello, use redis_cluster_address instead of redis_address in configuration. See https://centrifugal.dev/docs/server/engines#redis-cluster

Please reopen with updated details in case it still does not work.