[BUG] Connection to redis refused

vladislav06 commented 2 months ago

Describe the bug mailu-admin is unable to connect to mailu-redis-master :

Traceback (most recent call last):
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 276, in connect
    sock = self.retry.call_with_retry(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/retry.py", line 46, in call_with_retry
    return do()
           ^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 277, in <lambda>
    lambda: self._connect(), lambda error: self.disconnect(error)
            ^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 639, in _connect
    raise err
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 627, in _connect
    sock.connect(socket_address)
ConnectionRefusedError: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/app/venv/bin/flask", line 8, in <module>
    sys.exit(main())
             ^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 1105, in main
    cli.main()
  File "/app/venv/lib/python3.12/site-packages/click/core.py", line 1078, in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/click/core.py", line 1682, in invoke
    cmd_name, cmd, args = self.resolve_command(ctx, args)
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/click/core.py", line 1729, in resolve_command
    cmd = self.get_command(ctx, cmd_name)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 611, in get_command
    app = info.load_app()
          ^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 335, in load_app
    app = locate_app(import_name, name)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 262, in locate_app
    return find_best_app(module)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 72, in find_best_app
    app = app_factory()
          ^^^^^^^^^^^^^
  File "/app/mailu/__init__.py", line 117, in create_app
    return create_app_from_config(config)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/mailu/__init__.py", line 48, in create_app_from_config
    utils.session.init_app(app)
  File "/app/mailu/utils.py", line 497, in init_app
    MailuSessionExtension.cleanup_sessions(app)
  File "/app/mailu/utils.py", line 436, in cleanup_sessions
    for key in app.session_store.list():
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/mailu/utils.py", line 162, in list
    return list(self.redis.scan_iter(match=prefix))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/commands/core.py", line 3051, in scan_iter
    cursor, data = self.scan(
                   ^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/commands/core.py", line 3026, in scan
    return self.execute_command("SCAN", *pieces, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/client.py", line 542, in execute_command
    conn = self.connection or pool.get_connection(command_name, **options)
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 1109, in get_connection
    connection.connect()
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 282, in connect
    raise ConnectionError(self._error_message(e))
redis.exceptions.ConnectionError: Error 111 connecting to mailu-redis-master.mailu-mailserver.svc.cluster.local:6379. Connection refused.
Traceback (most recent call last):
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 276, in connect
    sock = self.retry.call_with_retry(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/retry.py", line 46, in call_with_retry
    return do()
           ^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 277, in <lambda>
    lambda: self._connect(), lambda error: self.disconnect(error)
            ^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 639, in _connect
    raise err
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 627, in _connect
    sock.connect(socket_address)
ConnectionRefusedError: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/app/venv/bin/flask", line 8, in <module>
    sys.exit(main())
             ^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 1105, in main
    cli.main()
  File "/app/venv/lib/python3.12/site-packages/click/core.py", line 1078, in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/click/core.py", line 1682, in invoke
    cmd_name, cmd, args = self.resolve_command(ctx, args)
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/click/core.py", line 1729, in resolve_command
    cmd = self.get_command(ctx, cmd_name)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 611, in get_command
    app = info.load_app()
          ^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 335, in load_app
    app = locate_app(import_name, name)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 262, in locate_app
    return find_best_app(module)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/flask/cli.py", line 72, in find_best_app
    app = app_factory()
          ^^^^^^^^^^^^^
  File "/app/mailu/__init__.py", line 117, in create_app
    return create_app_from_config(config)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/mailu/__init__.py", line 48, in create_app_from_config
    utils.session.init_app(app)
  File "/app/mailu/utils.py", line 497, in init_app
    MailuSessionExtension.cleanup_sessions(app)
  File "/app/mailu/utils.py", line 436, in cleanup_sessions
    for key in app.session_store.list():
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/mailu/utils.py", line 162, in list
    return list(self.redis.scan_iter(match=prefix))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/commands/core.py", line 3051, in scan_iter
    cursor, data = self.scan(
                   ^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/commands/core.py", line 3026, in scan
    return self.execute_command("SCAN", *pieces, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/client.py", line 542, in execute_command
    conn = self.connection or pool.get_connection(command_name, **options)
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 1109, in get_connection
    connection.connect()
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 282, in connect
    raise ConnectionError(self._error_message(e))
redis.exceptions.ConnectionError: Error 111 connecting to mailu-redis-master.mailu-mailserver.svc.cluster.local:6379. Connection refused.

Environment

bare metal k8s
metallb
nginx ingress

helm values:

global:
database:
roundcube:
  database: roundcube
  username: roundcube
commonLabels: {}
commonAnnotations: {}
affinity: {}
imageRegistry: ghcr.io
hostnames:
- "mail.blablabla.com"
domain: "blablabla.com"
secretKey: "************"
timezone: "Europe/Riga"
initialAccount:
enabled: true
username: "admin"
domain: "blablabla.com"
mode: "update"
api:
enabled: false
existingSecretTokenKey: "api-token"
webPath: "/api"
subnet: 10.244.0.0/16
networkPolicy:
enabled: true
ingressController:
namespace: ingress-nginx
podSelector: |
  matchLabels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/component: controller
logLevel: WARNING
postmaster: admin
recipientDelimiter: "+"
dmarc:
rua: "admin"
ruf: "admin"
limits:
messageSizeLimitInMegabytes: 50
authRatelimit:
ip: 60/hour
ipv4Mask: 24
ipv6Mask: 56
user: 100/day
exemptionLength: 86400
messageRatelimit:
value: 200/day
externalRelay:
usernameKey: "relay-username"
passwordKey: "relay-password"
clusterDomain: cluster.local
credentialRounds: 12
sessionCookieSecure: true
authRequireTokens: false
sessionTimeout: 3600
permanentSessionLifetime: 2592000
letsencryptShortchain: false
customization:
siteName: "mail"
website: "https://mail.blablabla.com"
welcomeMessage:
enabled: true
subject: "Welcome to Mailu"
body: "Welcome to Mailu, your new email service. Please change your password and update your profile."
externalDatabase:
enabled: false
port: 3306
database: mailu
username: mailu
existingSecretDatabaseKey: "database"
existingSecretUsernameKey: "username"
existingSecretPasswordKey: "password"
externalRedis:
enabled: false
port: 6379
adminQuotaDbId: 1
adminRateLimitDbId: 2
rspamdDbId: 0
database:
mysql:
postgresql:
persistence:
single_pvc: true
size: 100Gi
accessModes: [ReadWriteOnce]
annotations: {}
existingClaim: "mailu-nfs-pvc-mail"
ingress:
enabled: true
ingressClassName: "nginx"
pathType: ImplementationSpecific
path: /
annotations:
cert-manager.io/cluster-issuerr: "letsencrypt-prod"
tls: true
selfSigned: false
realIpHeader: X-Forwarded-For
proxyProtocol:
pop3: false
pop3s: false
imap: false
imaps: false
smtp: false
smtps: false
submission: false
manageSieve: false
proxyAuth:
header: "X-Auth-Email"
create: "false"
front:
image:
repository: mailu/nginx
pullPolicy: IfNotPresent
hostPort:
enabled: false
externalService:
enabled: true
type: LoadBalancer
externalTrafficPolicy: Local
annotations: {}
ports:
  pop3: false
  pop3s: true
  imap: false
  imaps: true
  smtp: true
  smtps: true
  submission: false
  manageSieve: true
kind: Deployment
replicaCount: 1
resources:
limits: {}
requests: {}
livenessProbe:
enabled: true
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
enabled: true
failureThreshold: 1
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
startupProbe:
enabled: false
failureThreshold: 30
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
podLabels: {}
podAnnotations: {}
nodeSelector: {}
podSecurityContext:
enabled: false
fsGroup: 1001
containerSecurityContext:
enabled: false
runAsUser: 1001
runAsNonRoot: false
terminationGracePeriodSeconds: 2
affinity: {}
revisionHistoryLimit: 3
service:
annotations: {}
updateStrategy:
type: RollingUpdate
admin:
enabled: true
uri: /admin
image:
repository: mailu/admin
pullPolicy: IfNotPresent
persistence:
size: 20Gi
accessModes: [ReadWriteOnce]
annotations: {}
resources:
limits: {}
requests: {}
livenessProbe:
enabled: true
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
enabled: true
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
startupProbe:
enabled: false
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
podLabels: {}
podAnnotations: {}
nodeSelector: {}
podSecurityContext:
enabled: false
fsGroup: 1001
containerSecurityContext:
enabled: false
runAsUser: 1001
runAsNonRoot: false
terminationGracePeriodSeconds: 2
affinity: {}
revisionHistoryLimit: 3
service:
annotations: {}
updateStrategy:
type: RollingUpdate
redis:
enabled: true
architecture: standalone
auth:
enabled: false
master:
enabled: true
count: 1
persistence:
  enabled: true
  size: 8Gi
  accessModes: [ReadWriteOnce]
  existingClaim: "mailu-nfs-pvc-redis"
  annotations: {}
replica:
count: 0

Additional context

WebSpider commented 2 months ago

Your redis service seems to be restarting / unavailable. Since k8s is about eventual consistency, it could take a while before your redis service becomes available.

If it takes a long time, look in the logs of the redis containers for hints on what could be the issue.

kaizerpwn commented 1 month ago

I'm experiencing a similar issue. Were you able to find a solution?

root@uspolodev:~# microk8s kubectl get pods -n mail
NAME                              READY   STATUS             RESTARTS      AGE
mailu-admin-6cc797dd56-q8sw9      0/1     CrashLoopBackOff   1 (17s ago)   46s
mailu-clamav-0                    1/1     Running            0             46s
mailu-dovecot-797cb79599-mf2jr    1/1     Running            0             46s
mailu-front-7b8c8dcd7b-72v8l      1/1     Running            0             46s
mailu-oletools-75d8774797-bck8k   1/1     Running            0             46s
mailu-postfix-6d67467c96-t7p6q    1/1     Running            0             46s
mailu-redis-master-0              1/1     Running            0             46s
mailu-rspamd-cdbb6d8f4-9qq6z      0/1     Running            0             46s
mailu-tika-dfb4ddbc6-zlqwv        1/1     Running            0             46s
mailu-webmail-84f956bc45-4sqzq    0/1     CrashLoopBackOff   2 (15s ago)   46s

root@uspolodev:~# microk8s kubectl logs mailu-admin-6cc797dd56-q8sw9 -n mail
Traceback (most recent call last):
  File "/app/venv/bin/gunicorn", line 8, in <module>
    sys.exit(run())
             ^^^^^
  File "/app/venv/lib/python3.12/site-packages/gunicorn/app/wsgiapp.py", line 67, in run
    WSGIApplication("%(prog)s [OPTIONS] [APP_MODULE]", prog=prog).run()
  File "/app/venv/lib/python3.12/site-packages/gunicorn/app/base.py", line 236, in run
    super().run()
  File "/app/venv/lib/python3.12/site-packages/gunicorn/app/base.py", line 72, in run
    Arbiter(self).run()
    ^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/gunicorn/arbiter.py", line 58, in __init__
    self.setup(app)
  File "/app/venv/lib/python3.12/site-packages/gunicorn/arbiter.py", line 118, in setup
    self.app.wsgi()
  File "/app/venv/lib/python3.12/site-packages/gunicorn/app/base.py", line 67, in wsgi
    self.callable = self.load()
                    ^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/gunicorn/app/wsgiapp.py", line 58, in load
    return self.load_wsgiapp()
           ^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/gunicorn/app/wsgiapp.py", line 48, in load_wsgiapp
    return util.import_app(self.app_uri)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/gunicorn/util.py", line 424, in import_app
    app = app(*args, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^
  File "/app/mailu/__init__.py", line 117, in create_app
    return create_app_from_config(config)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/mailu/__init__.py", line 48, in create_app_from_config
    utils.session.init_app(app)
  File "/app/mailu/utils.py", line 497, in init_app
    MailuSessionExtension.cleanup_sessions(app)
  File "/app/mailu/utils.py", line 436, in cleanup_sessions
    for key in app.session_store.list():
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/mailu/utils.py", line 162, in list
    return list(self.redis.scan_iter(match=prefix))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/commands/core.py", line 3051, in scan_iter
    cursor, data = self.scan(
                   ^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/commands/core.py", line 3026, in scan
    return self.execute_command("SCAN", *pieces, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/client.py", line 542, in execute_command
    conn = self.connection or pool.get_connection(command_name, **options)
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 1109, in get_connection
    connection.connect()
  File "/app/venv/lib/python3.12/site-packages/redis/connection.py", line 282, in connect
    raise ConnectionError(self._error_message(e))
redis.exceptions.ConnectionError: Error -5 connecting to mailu-redis-master.mail.svc.cluster.local:6379. Name has no usable address.

github-actions[bot] commented 1 day ago

This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.

Mailu / helm-charts

[BUG] Connection to redis refused #363