Closed giray closed 3 years ago
root@h2917067:/opt/mailcow-dockerized# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
02edaf19d16e robbertkl/ipv6nat "/docker-ipv6nat-com…" 13 minutes ago Up 13 minutes mailcowdockerized_ipv6nat-mailcow_1
2c3f3e8e085b mailcow/rspamd:1.77 "/docker-entrypoint.…" 13 minutes ago Up 13 minutes mailcowdockerized_rspamd-mailcow_1
43e838ef501a mcuadros/ofelia:latest "/usr/bin/ofelia dae…" 13 minutes ago Up 13 minutes mailcowdockerized_ofelia-mailcow_1
d0da256bdef9 mailcow/netfilter:1.43 "python3 -u /server.…" 13 minutes ago Up 13 minutes mailcowdockerized_netfilter-mailcow_1
dd32d27c3e8c mailcow/acme:1.79 "/sbin/tini -g -- /s…" 13 minutes ago Up 13 minutes mailcowdockerized_acme-mailcow_1
d332be2b28a3 mailcow/dovecot:1.153 "/docker-entrypoint.…" 13 minutes ago Up 13 minutes 0.0.0.0:110->110/tcp, :::110->110/tcp, 0.0.0.0:143->143/tcp, :::143->143/tcp, 0.0.0.0:993->993/tcp, :::993->993/tcp, 0.0.0.0:995->995/tcp, :::995->995/tcp, 0.0.0.0:4190->4190/tcp, :::4190->4190/tcp, 127.0.0.1:19991->12345/tcp mailcowdockerized_dovecot-mailcow_1
cafb6e79ecf9 mailcow/postfix:1.65 "/docker-entrypoint.…" 13 minutes ago Up 13 minutes 0.0.0.0:25->25/tcp, :::25->25/tcp, 0.0.0.0:465->465/tcp, :::465->465/tcp, 0.0.0.0:587->587/tcp, :::587->587/tcp, 588/tcp mailcowdockerized_postfix-mailcow_1
d919f6ddf07f nginx:mainline-alpine "/docker-entrypoint.…" 13 minutes ago Up 13 minutes 127.0.0.1:8080->8080/tcp, 80/tcp, 127.0.0.1:8443->8443/tcp mailcowdockerized_nginx-mailcow_1
bff4d3e88a7e mariadb:10.5 "docker-entrypoint.s…" 13 minutes ago Up 13 minutes 127.0.0.1:13306->3306/tcp mailcowdockerized_mysql-mailcow_1
c5d482b6dbbf mailcow/phpfpm:1.75 "/docker-entrypoint.…" 13 minutes ago Up 13 minutes 9000/tcp mailcowdockerized_php-fpm-mailcow_1
deeafef4b8ac memcached:alpine "docker-entrypoint.s…" 13 minutes ago Up 13 minutes 11211/tcp mailcowdockerized_memcached-mailcow_1
1433bde7fd6c mailcow/sogo:1.100 "/docker-entrypoint.…" 13 minutes ago Up 13 minutes mailcowdockerized_sogo-mailcow_1
13f6a3a2a0c0 mailcow/dockerapi:1.38 "python3 -u /app/doc…" 13 minutes ago Up 13 minutes mailcowdockerized_dockerapi-mailcow_1
53f1b9b7f42c redis:5-alpine "docker-entrypoint.s…" 13 minutes ago Up 13 minutes 127.0.0.1:7654->6379/tcp mailcowdockerized_redis-mailcow_1
d2c0c53c7ced mailcow/unbound:1.13 "/docker-entrypoint.…" 13 minutes ago Up 13 minutes 53/tcp, 53/udp mailcowdockerized_unbound-mailcow_1
ea25d43c6326 mailcow/solr:1.7 "docker-entrypoint.s…" 13 minutes ago Up 13 minutes 127.0.0.1:18983->8983/tcp mailcowdockerized_solr-mailcow_1
77aa459507a4 mailcow/watchdog:1.92 "/bin/sh -c '/watchd…" 13 minutes ago Up 13 minutes mailcowdockerized_watchdog-mailcow_1
3bed7e7dcf29 mailcow/clamd:1.40 "/sbin/tini -g -- /c…" 13 minutes ago Up 13 minutes mailcowdockerized_clamd-mailcow_1
36ce1bddf704 mailcow/olefy:1.7 "python3 -u /app/ole…" 13 minutes ago Up 13 minutes mailcowdockerized_olefy-mailcow_1
Hola, can you post the -t nat output? :) Any changes to the daemon.json?
of course here you go
root@h2917067:/opt/mailcow-dockerized# iptables -L -vn -t nat
Chain PREROUTING (policy ACCEPT 7028 packets, 420K bytes)
pkts bytes target prot opt in out source destination
1797K 95M DOCKER all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL
Chain INPUT (policy ACCEPT 7028 packets, 420K bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 4429 packets, 307K bytes)
pkts bytes target prot opt in out source destination
0 0 DOCKER all -- * * 0.0.0.0/0 !127.0.0.0/8 ADDRTYPE match dst-type LOCAL
Chain POSTROUTING (policy ACCEPT 4515 packets, 312K bytes)
pkts bytes target prot opt in out source destination
0 0 MASQUERADE all -- * !br-mailcow 172.22.1.0/24 0.0.0.0/0
0 0 MASQUERADE all -- * !docker0 172.17.0.0/16 0.0.0.0/0
0 0 MASQUERADE tcp -- * * 172.22.1.249 172.22.1.249 tcp dpt:6379
0 0 MASQUERADE tcp -- * * 172.22.1.4 172.22.1.4 tcp dpt:8983
0 0 MASQUERADE tcp -- * * 172.22.1.9 172.22.1.9 tcp dpt:3306
0 0 MASQUERADE tcp -- * * 172.22.1.10 172.22.1.10 tcp dpt:8443
0 0 MASQUERADE tcp -- * * 172.22.1.10 172.22.1.10 tcp dpt:8080
0 0 MASQUERADE tcp -- * * 172.22.1.253 172.22.1.253 tcp dpt:587
0 0 MASQUERADE tcp -- * * 172.22.1.250 172.22.1.250 tcp dpt:12345
0 0 MASQUERADE tcp -- * * 172.22.1.250 172.22.1.250 tcp dpt:4190
0 0 MASQUERADE tcp -- * * 172.22.1.253 172.22.1.253 tcp dpt:465
0 0 MASQUERADE tcp -- * * 172.22.1.250 172.22.1.250 tcp dpt:995
0 0 MASQUERADE tcp -- * * 172.22.1.253 172.22.1.253 tcp dpt:25
0 0 MASQUERADE tcp -- * * 172.22.1.250 172.22.1.250 tcp dpt:993
0 0 MASQUERADE tcp -- * * 172.22.1.250 172.22.1.250 tcp dpt:143
0 0 MASQUERADE tcp -- * * 172.22.1.250 172.22.1.250 tcp dpt:110
Chain DOCKER (2 references)
pkts bytes target prot opt in out source destination
0 0 RETURN all -- br-mailcow * 0.0.0.0/0 0.0.0.0/0
0 0 RETURN all -- docker0 * 0.0.0.0/0 0.0.0.0/0
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 127.0.0.1 tcp dpt:7654 to:172.22.1.249:6379
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 127.0.0.1 tcp dpt:18983 to:172.22.1.4:8983
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 127.0.0.1 tcp dpt:13306 to:172.22.1.9:3306
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 127.0.0.1 tcp dpt:8443 to:172.22.1.10:8443
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 127.0.0.1 tcp dpt:8080 to:172.22.1.10:8080
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 0.0.0.0/0 tcp dpt:587 to:172.22.1.253:587
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 127.0.0.1 tcp dpt:19991 to:172.22.1.250:12345
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 0.0.0.0/0 tcp dpt:4190 to:172.22.1.250:4190
48 2880 DNAT tcp -- !br-mailcow * 0.0.0.0/0 0.0.0.0/0 tcp dpt:465 to:172.22.1.253:465
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 0.0.0.0/0 tcp dpt:995 to:172.22.1.250:995
38 2252 DNAT tcp -- !br-mailcow * 0.0.0.0/0 0.0.0.0/0 tcp dpt:25 to:172.22.1.253:25
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 0.0.0.0/0 tcp dpt:993 to:172.22.1.250:993
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 0.0.0.0/0 tcp dpt:143 to:172.22.1.250:143
0 0 DNAT tcp -- !br-mailcow * 0.0.0.0/0 0.0.0.0/0 tcp dpt:110 to:172.22.1.250:110
root@h2917067:/opt/mailcow-dockerized#
... and I do not seem to have a daemon.json
setup is pretty vanilla ... and has been working great for the past two years
Can you try to run docker-compose down
, remove the change to docker-compose.yml, systemctl restart docker, docker-compose up -d
?
done ..
# diff docker-compose.yml __docker-compose.yml
607c607,610
< com.docker.network.bridge.name: br-mailcow
---
> com.docker.network.bridge.name: "br-mailcow"
> com.docker.network.bridge.enable_ip_masquerade: "true"
> com.docker.network.bridge.host_binding_ipv4": "0.0.0.0"
redis seems up
mailcowdockerized_redis-mailcow_1 mailcowdockerized_rspamd-mailcow_1
root@h2917067:/opt/mailcow-dockerized# docker logs -f mailcowdockerized_redis-mailcow_1
1:C 01 Jul 2021 15:56:08.573 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
1:C 01 Jul 2021 15:56:08.573 # Redis version=5.0.12, bits=64, commit=00000000, modified=0, pid=1, just started
1:C 01 Jul 2021 15:56:08.573 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf
1:M 01 Jul 2021 15:56:08.574 * Running mode=standalone, port=6379.
1:M 01 Jul 2021 15:56:08.574 # Server initialized
1:M 01 Jul 2021 15:56:08.574 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
1:M 01 Jul 2021 15:56:08.574 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled.
1:M 01 Jul 2021 15:56:08.645 * DB loaded from disk: 0.071 seconds
1:M 01 Jul 2021 15:56:08.645 * Ready to accept connections
but ....
acme-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
acme-mailcow_1 | Waiting for Redis...
php-fpm-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
php-fpm-mailcow_1 | Waiting for Redis...
acme-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
acme-mailcow_1 | Waiting for Redis...
php-fpm-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
php-fpm-mailcow_1 | Waiting for Redis...
sogo-mailcow_1 | Waiting for schema update...
rspamd-mailcow_1 | Waiting for PHP on port 9001...
acme-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
acme-mailcow_1 | Waiting for Redis...
php-fpm-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
php-fpm-mailcow_1 | Waiting for Redis...
dovecot-mailcow_1 | Waiting for DNS...
rspamd-mailcow_1 | Waiting for PHP on port 9001...
postfix-mailcow_1 | Waiting for DNS...
acme-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
acme-mailcow_1 | Waiting for Redis...
php-fpm-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
php-fpm-mailcow_1 | Waiting for Redis...
sogo-mailcow_1 | Waiting for schema update...
netfilter-mailcow_1 | Error 110 connecting to 172.22.1.249:6379. Operation timed out. - trying again in 3 seconds
acme-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
acme-mailcow_1 | Waiting for Redis...
php-fpm-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
php-fpm-mailcow_1 | Waiting for Redis...
You might want to try to reset all iptables rules (flush them). I have never seen this happening. It is obviously a network issue, but I cannot tell you what and where to look for.
rebooted the box ... no joy ... still hanging on
acme-mailcow_1 | Waiting for Redis...
php-fpm-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable
php-fpm-mailcow_1 | Waiting for Redis...
sogo-mailcow_1 | Waiting for schema update...
postfix-mailcow_1 | Waiting for DNS...
rspamd-mailcow_1 | Waiting for PHP on port 9001...
dovecot-mailcow_1 | Waiting for DNS...
If you’re able to join the Telegram channel, we can surely find a way for me to have a quick glance at it.
Am 01.07.2021 um 18:06 schrieb giray @.***>:
rebooted the box ... no joy ... still hanging on
acme-mailcow_1 | Waiting for Redis... php-fpm-mailcow_1 | Could not connect to Redis at redis:6379: Host is unreachable php-fpm-mailcow_1 | Waiting for Redis... sogo-mailcow_1 | Waiting for schema update... postfix-mailcow_1 | Waiting for DNS... rspamd-mailcow_1 | Waiting for PHP on port 9001... dovecot-mailcow_1 | Waiting for DNS... — You are receiving this because you commented. Reply to this email directly, view it on GitHub, or unsubscribe.
@andryyy sure ... im @giraydevlet now on the mailcow channel
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs.
Prior to placing the issue, please check following: (fill out each checkbox with an
X
once done)Summary
After upgrade to latest version for
update.sh
redis and dns are unavailable and the stack will not start upLogs
Reproduction
System information
docker version
)docker-compose version
)git diff origin/master
, any other changes to the code? If so, please post them.enable_ipv6: true ipam: driver: default
iptables -L -vn
,ip6tables -L -vn
,iptables -L -vn -t nat
andip6tables -L -vn -t nat
.Chain FORWARD (policy ACCEPT 45 packets, 2652 bytes) pkts bytes target prot opt in out source destination
46 2994 DOCKER-USER all -- 0.0.0.0/0 0.0.0.0/0
46 2994 DOCKER-ISOLATION-STAGE-1 all -- 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- br-mailcow 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED 1 342 DOCKER all -- br-mailcow 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- br-mailcow !br-mailcow 0.0.0.0/0 0.0.0.0/0
1 342 ACCEPT all -- br-mailcow br-mailcow 0.0.0.0/0 0.0.0.0/0
20M 16G MAILCOW all -- 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED 0 0 DOCKER all -- docker0 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 docker0 0.0.0.0/0 0.0.0.0/0
Chain OUTPUT (policy ACCEPT 30562 packets, 56M bytes) pkts bytes target prot opt in out source destination
Chain DOCKER (2 references) pkts bytes target prot opt in out source destination
0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.249 tcp dpt:6379 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.4 tcp dpt:8983 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.9 tcp dpt:3306 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.10 tcp dpt:8443 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.10 tcp dpt:8080 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.253 tcp dpt:587 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.250 tcp dpt:12345 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.250 tcp dpt:4190 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.253 tcp dpt:465 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.250 tcp dpt:995 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.253 tcp dpt:25 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.250 tcp dpt:993 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.250 tcp dpt:143 0 0 ACCEPT tcp -- !br-mailcow br-mailcow 0.0.0.0/0 172.22.1.250 tcp dpt:110
Chain DOCKER-ISOLATION-STAGE-1 (1 references) pkts bytes target prot opt in out source destination
0 0 DOCKER-ISOLATION-STAGE-2 all -- br-mailcow !br-mailcow 0.0.0.0/0 0.0.0.0/0
0 0 DOCKER-ISOLATION-STAGE-2 all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
20M 16G RETURN all -- 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-ISOLATION-STAGE-2 (2 references) pkts bytes target prot opt in out source destination
0 0 DROP all -- br-mailcow 0.0.0.0/0 0.0.0.0/0
0 0 DROP all -- docker0 0.0.0.0/0 0.0.0.0/0
2422K 386M RETURN all -- 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-USER (1 references) pkts bytes target prot opt in out source destination
98M 92G RETURN all -- 0.0.0.0/0 0.0.0.0/0
Chain MAILCOW (2 references) pkts bytes target prot opt in out source destination
root@h2917067:/opt/mailcow-dockerized#
root@h2917067:/opt/mailcow-dockerized# docker exec -it $(docker ps -qf name=acme-mailcow) dig +short stackoverflow.com @172.22.1.254 ;; connection timed out; no servers could be reached