szaimen
commented
1 year ago Hi, thanks for the report! Can you post the mastercontainer logs here?
Closed MariusQuabeck closed 1 year ago
szaimen
commented
1 year ago Hi, thanks for the report! Can you post the mastercontainer logs here?
MariusQuabeck
commented
1 year ago mastercontainer log
szaimen
commented
1 year ago Thanks! Can you post the output of the following command?
sudo netstat -tulpn
MariusQuabeck
commented
1 year ago Do I need to run that on host or in some container? host (linode app/ubuntu 22.04) doesn't have a package named "netstat", I had to install "netstat-nat" but that one doesn't like your command :)
root:~# sudo netstat-nat -tulpn
netstat-nat: invalid option -- 't'You need to install it with sudo apt install net-tools
MariusQuabeck
commented
1 year ago ah my bad :D
root~# sudo netstat -tulpn
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 1232/docker-proxy
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 753/sshd: /usr/sbin
tcp 0 0 0.0.0.0:8443 0.0.0.0:* LISTEN 1005/docker-proxy
tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN 1149/docker-proxy
tcp 0 0 0.0.0.0:3478 0.0.0.0:* LISTEN 1099/docker-proxy
tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 543/systemd-resolve
tcp6 0 0 :::80 :::* LISTEN 1243/docker-proxy
tcp6 0 0 :::22 :::* LISTEN 753/sshd: /usr/sbin
tcp6 0 0 :::8443 :::* LISTEN 1018/docker-proxy
tcp6 0 0 :::8080 :::* LISTEN 1155/docker-proxy
tcp6 0 0 :::3478 :::* LISTEN 1105/docker-proxy
udp 0 0 0.0.0.0:3478 0.0.0.0:* 1196/docker-proxy
udp 0 0 127.0.0.53:53 0.0.0.0:* 543/systemd-resolve
udp6 0 0 :::3478 :::* 1201/docker-proxy Thanks! What happens if you want to check the logs of the apache container? What do you see then?
MariusQuabeck
commented
1 year ago :8443/api/docker/logs?id=nextcloud-aio-apache gives me a completely empty page :/
szaimen
commented
1 year ago Is there a new log line in the mastercontainer logs after attempting to open the apache logs?
MariusQuabeck
commented
1 year ago yes!
szaimen
commented
1 year ago What is the output of sudo docker inspect nextcloud-aio-apache ?
MariusQuabeck
commented
1 year ago sudo docker inspect nextcloud-aio-apache
[
{
"Id": "c95fb36069630f5ed5736ba5e15720707e50dc0e6cb3c03f1ab9bb0c58b33c35",
"Created": "2023-09-20T06:45:09.573135766Z",
"Path": "/start.sh",
"Args": [
"/usr/bin/supervisord",
"-c",
"/supervisord.conf"
],
"State": {
"Status": "created",
"Running": false,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 0,
"ExitCode": 128,
"Error": "driver failed programming external connectivity on endpoint nextcloud-aio-apache (ca03df1c41a428c37a38e6f6bb144e7398d2ed4ae078e415a0d79ee5ae08d759): Bind for 0.0.0.0:443 failed: port is already allocated",
"StartedAt": "0001-01-01T00:00:00Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:e55d7dcd9586f3493ddd0d80684fd3446c7883b00a6974743a3ee2cc7d9c027d",
"ResolvConfPath": "/var/lib/docker/containers/c95fb36069630f5ed5736ba5e15720707e50dc0e6cb3c03f1ab9bb0c58b33c35/resolv.conf",
"HostnamePath": "",
"HostsPath": "/var/lib/docker/containers/c95fb36069630f5ed5736ba5e15720707e50dc0e6cb3c03f1ab9bb0c58b33c35/hosts",
"LogPath": "/var/lib/docker/containers/c95fb36069630f5ed5736ba5e15720707e50dc0e6cb3c03f1ab9bb0c58b33c35/c95fb36069630f5ed5736ba5e15720707e50dc0e6cb3c03f1ab9bb0c58b33c35-json.log",
"Name": "/nextcloud-aio-apache",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"nextcloud_aio_nextcloud:/var/www/html:ro",
"nextcloud_aio_apache:/mnt/data:rw"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "nextcloud-aio",
"PortBindings": {
"443/tcp": [
{
"HostIp": "",
"HostPort": "443"
}
],
"443/udp": [
{
"HostIp": "",
"HostPort": "443"
}
]
},
"RestartPolicy": {
"Name": "unless-stopped",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"ConsoleSize": [
0,
0
],
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "private",
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": true,
"SecurityOpt": null,
"Tmpfs": {
"/home/www-data": "",
"/tmp": "",
"/usr/local/apache2/logs": "",
"/var/log/supervisord": "",
"/var/run/supervisord": ""
},
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DeviceRequests": null,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": null,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
],
"Init": true
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/8de8687d4daa9b20eacbf4747151f4a261c21fdd9a01059a006350b3d9b60787-init/diff:/var/lib/docker/overlay2/4756e0fcfda4b00c7f9796232dba0cb1bbd6f0d48db5195dc356d4b8074b9e46/diff:/var/lib/docker/overlay2/7056fb227533cb547880f0159fbed0f3b7715c36ca2ea3fa2823599190528b57/diff:/var/lib/docker/overlay2/a156aaa6b3137b4f6af351d526a53902c660f871d15a28dccf09e0b5d1d31296/diff:/var/lib/docker/overlay2/60d1e3341e16ec1c32f4e9160c76b41b50a2125c4deaf0f220c79ab955b5bf57/diff:/var/lib/docker/overlay2/309c5635425f414b3539067374887f0b20956836ab11570c680f85b6bad99bdf/diff:/var/lib/docker/overlay2/90d2578acfead9acb0cded8ea6a5034d5d9f521076f6f3574f1e61a0850de8d2/diff:/var/lib/docker/overlay2/d249b1ddfe239f1103862ec35c0f3734c466e9d90785330c0f7cb0e9ea17d58a/diff:/var/lib/docker/overlay2/014c14ae024968f97547c4923f0dfec6133d5240668772d7e891e5cc470612f1/diff:/var/lib/docker/overlay2/4694f5815b056471da08d2ca45acab97fcee8b7626b8f0b2a88588f1ffa1c71a/diff:/var/lib/docker/overlay2/c62aaedee2c39c19e39bf761a4c00215ba5e31b3d9d395bf71ba0ad57673f910/diff:/var/lib/docker/overlay2/45517f0aa966857a4deb8e31fbb34b0cadeaf7ee2db4943dc0f3cb263f46418f/diff:/var/lib/docker/overlay2/78edbc5013cb18201e30fb02d8f059f2b6bf1e110bec7e27897eae458371e22d/diff:/var/lib/docker/overlay2/0f7a20c7284be376855dc0c00c67758047b32ae7432e7e8d0c41c307150df902/diff",
"MergedDir": "/var/lib/docker/overlay2/8de8687d4daa9b20eacbf4747151f4a261c21fdd9a01059a006350b3d9b60787/merged",
"UpperDir": "/var/lib/docker/overlay2/8de8687d4daa9b20eacbf4747151f4a261c21fdd9a01059a006350b3d9b60787/diff",
"WorkDir": "/var/lib/docker/overlay2/8de8687d4daa9b20eacbf4747151f4a261c21fdd9a01059a006350b3d9b60787/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "nextcloud_aio_apache",
"Source": "/var/lib/docker/volumes/nextcloud_aio_apache/_data",
"Destination": "/mnt/data",
"Driver": "local",
"Mode": "rw",
"RW": true,
"Propagation": ""
},
{
"Type": "volume",
"Name": "nextcloud_aio_nextcloud",
"Source": "/var/lib/docker/volumes/nextcloud_aio_nextcloud/_data",
"Destination": "/var/www/html",
"Driver": "local",
"Mode": "ro",
"RW": false,
"Propagation": ""
}
],
"Config": {
"Hostname": "c95fb3606963",
"Domainname": "",
"User": "www-data",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"443/tcp": {},
"443/udp": {},
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"NC_DOMAIN=cloud.nerdzoom.media",
"NEXTCLOUD_HOST=nextcloud-aio-nextcloud",
"COLLABORA_HOST=nextcloud-aio-collabora",
"TALK_HOST=nextcloud-aio-talk",
"APACHE_PORT=443",
"ONLYOFFICE_HOST=nextcloud-aio-onlyoffice",
"TZ=Etc/UTC",
"APACHE_MAX_SIZE=10737418240",
"APACHE_MAX_TIME=3600",
"NOTIFY_PUSH_HOST=nextcloud-aio-notify-push",
"PATH=/usr/local/apache2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"HTTPD_PREFIX=/usr/local/apache2",
"HTTPD_VERSION=2.4.57",
"HTTPD_SHA256=dbccb84aee95e095edfbb81e5eb926ccd24e6ada55dcd83caecb262e5cf94d2a",
"HTTPD_PATCHES=rewrite-windows-testchar-h.patch 1d5620574fa03b483262dc5b9a66a6906553389952ab5d3070a02f887cc20193"
],
"Cmd": [
"/usr/bin/supervisord",
"-c",
"/supervisord.conf"
],
"Healthcheck": {
"Test": [
"CMD-SHELL",
"/healthcheck.sh"
]
},
"Image": "nextcloud/aio-apache:latest",
"Volumes": {
"/mnt/data": {}
},
"WorkingDir": "/usr/local/apache2",
"Entrypoint": [
"/start.sh"
],
"OnBuild": null,
"Labels": {
"com.centurylinklabs.watchtower.enable": "false"
},
"StopSignal": "SIGWINCH"
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "f39c4a34cbd6a39e651c0fc9299ffccf278763273c9cca80502618cf312bf28e",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "/var/run/docker/netns/f39c4a34cbd6",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"nextcloud-aio": {
"IPAMConfig": null,
"Links": null,
"Aliases": [
"c95fb3606963"
],
"NetworkID": "678e7e24130f1c6c32d57fa75b1ab8028aca63ba174e65571b0192fbeb6e2cda",
"EndpointID": "ca03df1c41a428c37a38e6f6bb144e7398d2ed4ae078e415a0d79ee5ae08d759",
"Gateway": "",
"IPAddress": "172.18.0.10",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:12:00:0a",
"DriverOpts": null
}
}
}
}
]
szaimen
commented
1 year ago Finally we found the issue:
"Error": "driver failed programming external connectivity on endpoint nextcloud-aio-apache (ca03df1c41a428c37a38e6f6bb144e7398d2ed4ae078e415a0d79ee5ae08d759): Bind for 0.0.0.0:443 failed: port is already allocated",Can you maybe restart your server and try to start the containers again? Does it work afterwards?
MariusQuabeck
commented
1 year ago after rebooting the initial container start worked except for apache, then after stopping and starting containers again, it worked.
szaimen
commented
1 year ago So I suspect that the firwall caused this... Sometimes the server still needs to be restarted unfortunately...
Apache itself is not giving any logs, here are the logs that I have:
Database
Setting max connections... chmod: /var/run/postgresql: Operation not permitted PostgreSQL Database directory appears to contain a database; Skipping initialization 2023-09-20 06:45:02.070 UTC [14] LOG: starting PostgreSQL 15.4 on x86_64-pc-linux-musl, compiled by gcc (Alpine 12.2.1_git20220924-r10) 12.2.1 20220924, 64-bit 2023-09-20 06:45:02.070 UTC [14] LOG: listening on IPv4 address "0.0.0.0", port 5432 2023-09-20 06:45:02.070 UTC [14] LOG: listening on IPv6 address "::", port 5432 2023-09-20 06:45:02.072 UTC [14] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" 2023-09-20 06:45:02.078 UTC [24] LOG: database system was shut down at 2023-09-20 06:44:51 UTC 2023-09-20 06:45:02.087 UTC [14] LOG: database system is ready to accept connections 2023-09-20 06:47:11.284 UTC [14] LOG: received fast shutdown request 2023-09-20 06:47:11.290 UTC [14] LOG: aborting any active transactions 2023-09-20 06:47:11.294 UTC [14] LOG: background worker "logical replication launcher" (PID 27) exited with exit code 1 2023-09-20 06:47:11.301 UTC [22] LOG: shutting down 2023-09-20 06:47:11.316 UTC [14] LOG: database system is shut down waiting for server to shut down.... done server stopped Database dump successful! Setting max connections... chmod: /var/run/postgresql: Operation not permitted PostgreSQL Database directory appears to contain a database; Skipping initialization 2023-09-20 06:49:18.861 UTC [14] LOG: starting PostgreSQL 15.4 on x86_64-pc-linux-musl, compiled by gcc (Alpine 12.2.1_git20220924-r10) 12.2.1 20220924, 64-bit 2023-09-20 06:49:18.861 UTC [14] LOG: listening on IPv4 address "0.0.0.0", port 5432 2023-09-20 06:49:18.861 UTC [14] LOG: listening on IPv6 address "::", port 5432 2023-09-20 06:49:18.863 UTC [14] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" 2023-09-20 06:49:18.876 UTC [24] LOG: database system was shut down at 2023-09-20 06:47:11 UTC 2023-09-20 06:49:18.894 UTC [14] LOG: database system is ready to accept connectionsNextcloud
now ------------------------------- 2023-09-20 06:45:09.363487+00 (1 row) + '[' -f /dev-dri-group-was-added ']' ++ find /dev -maxdepth 1 -mindepth 1 -name dri + '[' -n '' ']' + set +x Installing imagemagick via apk... Enabling Imagick... Configuring Redis as session handler... Setting php max children... Applying one-click-instance settings... System config value one-click-instance set to boolean true System config value one-click-instance.user-limit set to integer 100 System config value one-click-instance.link set to string https://nextcloud.com/all-in-one/ support already enabled Adjusting log files... System config value upgrade.cli-upgrade-link set to string https://github.com/nextcloud/all-in-one/discussions/2726 System config value logfile set to string /var/www/html/data/nextcloud.log Config value logfile for app admin_audit set to /var/www/html/data/audit.log System config value updatedirectory set to string /nc-updater Applying network settings... System config value davstorage.request_timeout set to integer 3600 System config value trusted_domains => 1 set to string cloud.nerdzoom.media System config value overwrite.cli.url set to string https://cloud.nerdzoom.media/ System config value htaccess.RewriteBase set to string / .htaccess has been updated System config value dbpersistent set to boolean true System config value files_external_allow_create_new_local set to boolean true System config value trusted_proxies => 0 set to string 127.0.0.1 System config value trusted_proxies => 1 set to string ::1 Config value base_endpoint for app notify_push set to https://cloud.nerdzoom.media/push Config value wopi_url for app richdocuments set to https://cloud.nerdzoom.media/ System config value allow_local_remote_servers set to boolean true Config value wopi_allowlist for app richdocuments set to 172.105.131.67,2a01:7e01::f03c:93ff:fe6c:bf05,127.0.0.1/8,192.168.0.0/16,172.16.0.0/12,10.0.0.0/8,fd00::/8,::1 Config value recording_servers of app spreed deleted System config value enabledPreviewProviders => 0 set to string OC\Preview\Imaginary System config value preview_imaginary_url set to string http://nextcloud-aio-imaginary:9000 Waiting for database to start... now ------------------------------- 2023-09-20 06:49:25.748641+00 (1 row) + '[' -f /dev-dri-group-was-added ']' ++ find /dev -maxdepth 1 -mindepth 1 -name dri + '[' -n '' ']' + set +x Configuring Redis as session handler... Setting php max children... Applying one-click-instance settings... System config value one-click-instance set to boolean true System config value one-click-instance.user-limit set to integer 100 System config value one-click-instance.link set to string https://nextcloud.com/all-in-one/ support already enabled Adjusting log files... System config value upgrade.cli-upgrade-link set to string https://github.com/nextcloud/all-in-one/discussions/2726 System config value logfile set to string /var/www/html/data/nextcloud.log Config value logfile for app admin_audit set to /var/www/html/data/audit.log System config value updatedirectory set to string /nc-updater Applying network settings... System config value davstorage.request_timeout set to integer 3600 System config value trusted_domains => 1 set to string cloud.nerdzoom.media System config value overwrite.cli.url set to string https://cloud.nerdzoom.media/ System config value htaccess.RewriteBase set to string / .htaccess has been updated System config value dbpersistent set to boolean true System config value files_external_allow_create_new_local set to boolean true System config value trusted_proxies => 0 set to string 127.0.0.1 System config value trusted_proxies => 1 set to string ::1 Config value base_endpoint for app notify_push set to https://cloud.nerdzoom.media/push Config value wopi_url for app richdocuments set to https://cloud.nerdzoom.media/ System config value allow_local_remote_servers set to boolean true Config value wopi_allowlist for app richdocuments set to 172.105.131.67,2a01:7e01::f03c:93ff:fe6c:bf05,127.0.0.1/8,192.168.0.0/16,172.16.0.0/12,10.0.0.0/8,fd00::/8,::1 Config value recording_servers of app spreed deleted System config value enabledPreviewProviders => 0 set to string OC\Preview\Imaginary System config value preview_imaginary_url set to string http://nextcloud-aio-imaginary:9000 [20-Sep-2023 06:50:16] NOTICE: fpm is running, pid 367 [20-Sep-2023 06:50:16] NOTICE: ready to handle connectionsThis issue started a few weeks ago, Nextcloud was unreachable in the morning, I assume after Apache not starting after automatic backup/update. It usually came back after a few tries of restarting all containers.