search

docker / for-linux

Docker Engine for Linux
https://docs.docker.com/engine/installation/
754 stars 85 forks source link

Docker swarm not configuring default gateway on containers connected to overlay network #1432

Open FairyTail2000 opened 2 years ago

FairyTail2000 commented 2 years ago 
docker system info output on the manager:
Server:
 Containers: 4
  Running: 1
  Paused: 0
  Stopped: 3
 Images: 7
 Server Version: 20.10.17
 Storage Driver: overlay2
  Backing Filesystem: extfs
  Supports d_type: true
  Native Overlay Diff: true
  userxattr: false
 Logging Driver: json-file
 Cgroup Driver: systemd
 Cgroup Version: 2
 Plugins:
  Volume: local
  Network: bridge host ipvlan macvlan null overlay
  Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
 Swarm: active
  NodeID: lbwupro0h6kjs34p7gy2603m0
  Is Manager: true
  ClusterID: llsw4ilfu5xjedadjwlbtz4ff
  Managers: 1
  Nodes: 2
  Default Address Pool: 10.0.0.0/8  
  SubnetSize: 24
  Data Path Port: 4789
  Orchestration:
   Task History Retention Limit: 5
  Raft:
   Snapshot Interval: 10000
   Number of Old Snapshots to Retain: 0
   Heartbeat Tick: 1
   Election Tick: 10
  Dispatcher:
   Heartbeat Period: 5 seconds
  CA Configuration:
   Expiry Duration: 3 months
   Force Rotate: 0
  Autolock Managers: false
  Root Rotation In Progress: false
  Node Address: X.X.X.X
  Manager Addresses:
   X.X.X.X:2377
 Runtimes: io.containerd.runc.v2 io.containerd.runtime.v1.linux runc
 Default Runtime: runc
 Init Binary: docker-init
 containerd version: 0197261a30bf81f1ee8e6a4dd2dea0ef95d67ccb
 runc version: v1.1.3-0-g6724737
 init version: de40ad0
 Security Options:
  apparmor
  seccomp
   Profile: default
  cgroupns
 Kernel Version: 5.10.0-17-amd64
 Operating System: Debian GNU/Linux 11 (bullseye)
 OSType: linux
 Architecture: x86_64
 CPUs: 6
 Total Memory: 15.63GiB
 Name: XXXXXXXXXXXXXXXXXX
 ID: HUIY:DTNX:UDBR:2RNR:KFNE:J2WQ:J3RL:ZV3E:QLCP:T6WR:7QW6:M2DD
 Docker Root Dir: /var/lib/docker
 Debug Mode: false
 Registry: https://index.docker.io/v1/
 Labels:
 Experimental: false
 Insecure Registries:
  127.0.0.0/8
 Live Restore Enabled: false
docker inspect traefik_traefik-network
[
    {
        "Name": "traefik_traefik-network",
        "Id": "gre6shjq24dnp3lb20hajeyxl",
        "Created": "2022-08-17T21:09:46.857818326+02:00",
        "Scope": "swarm",
        "Driver": "overlay",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "10.0.2.0/24",
                    "Gateway": "10.0.2.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": false,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {
            "853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee": {
                "Name": "reverse-proxy_reverse-proxy.1.n8dxvft590laao30f7y1zspwe",
                "EndpointID": "532d9f9cc1d1127d4e03d5052054c7b96f8131c6b74e5d748bdf69323a7cacd5",
                "MacAddress": "02:42:0a:00:02:04",
                "IPv4Address": "10.0.2.4/24",
                "IPv6Address": ""
            },
            "lb-traefik_traefik-network": {
                "Name": "traefik_traefik-network-endpoint",
                "EndpointID": "1bb5b3c913466a8de526bc5158cfc54f808fda5c01e10614660f4cdd799bafe9",
                "MacAddress": "02:42:0a:00:02:05",
                "IPv4Address": "10.0.2.5/24",
                "IPv6Address": ""
            }
        },
        "Options": {
            "com.docker.network.driver.overlay.vxlanid_list": "4098"
        },
        "Labels": {
            "com.docker.stack.namespace": "reverse-proxy"
        },
        "Peers": [
            {
                "Name": "8d376eb566f8",
                "IP": "X.X.X.X"
            },
            {
                "Name": "372bc9b0c934",
                "IP": "Y.Y.Y.Y"
            }
        ]
    }
]
docker container inspect reverse-proxy_reverse-proxy.1.n8dxvft590laao30f7y1zspwe
[
    {
        "Id": "853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee",
        "Created": "2022-08-17T19:09:43.013601794Z",
        "Path": "/entrypoint.sh",
        "Args": [
            "traefik"
        ],
        "State": {
            "Status": "running",
            "Running": true,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 4223,
            "ExitCode": 0,
            "Error": "",
            "StartedAt": "2022-08-17T19:09:48.193302818Z",
            "FinishedAt": "0001-01-01T00:00:00Z"
        },
        "Image": "sha256:22c6901de2be6ce40b6c8b10426a5617c3a8715f78164f2408e94fe34e55110d",
        "ResolvConfPath": "/var/lib/docker/containers/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee/hostname",
        "HostsPath": "/var/lib/docker/containers/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee/hosts",
        "LogPath": "/var/lib/docker/containers/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee-json.log",
        "Name": "/reverse-proxy_reverse-proxy.1.n8dxvft590laao30f7y1zspwe",
        "RestartCount": 0,
        "Driver": "overlay2",
        "Platform": "linux",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "docker-default",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": null,
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {
                    "max-file": "2",
                    "max-size": "50m",
                    "tag": "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}|{{index .ContainerLabels \"com.docker.compose.project\"}}|{{index .ContainerLabels \"com.docker.compose.service\"}}"
                }
            },
            "NetworkMode": "default",
            "PortBindings": {
                "443/tcp": [
                    {
                        "HostIp": "",
                        "HostPort": "443"
                    }
                ],
                "5432/tcp": [
                    {
                        "HostIp": "",
                        "HostPort": "5432"
                    }
                ],
                "80/tcp": [
                    {
                        "HostIp": "",
                        "HostPort": "80"
                    }
                ]
            },
            "RestartPolicy": {
                "Name": "",
                "MaximumRetryCount": 0
            },
            "AutoRemove": false,
            "VolumeDriver": "",
            "VolumesFrom": null,
            "CapAdd": null,
            "CapDrop": null,
            "CgroupnsMode": "private",
            "Dns": null,
            "DnsOptions": null,
            "DnsSearch": null,
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "private",
            "Cgroup": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "UsernsMode": "",
            "ShmSize": 67108864,
            "Runtime": "runc",
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "default",
            "CpuShares": 0,
            "Memory": 157286400,
            "NanoCpus": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpuRealtimePeriod": 0,
            "CpuRealtimeRuntime": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": null,
            "DeviceCgroupRules": null,
            "DeviceRequests": null,
            "KernelMemory": 0,
            "KernelMemoryTCP": 0,
            "MemoryReservation": 0,
            "MemorySwap": 314572800,
            "MemorySwappiness": null,
            "OomKillDisable": null,
            "PidsLimit": null,
            "Ulimits": [],
            "CpuCount": 0,
            "CpuPercent": 0,
            "IOMaximumIOps": 0,
            "IOMaximumBandwidth": 0,
            "Mounts": [
                {
                    "Type": "bind",
                    "Source": "/var/run/docker.sock",
                    "Target": "/var/run/docker.sock",
                    "ReadOnly": true
                },
                {
                    "Type": "volume",
                    "Source": "reverse-proxy_certificates",
                    "Target": "/acme",
                    "VolumeOptions": {
                        "Labels": {
                            "com.docker.stack.namespace": "reverse-proxy"
                        }
                    }
                }
            ],
            "MaskedPaths": [
                "/proc/asound",
                "/proc/acpi",
                "/proc/kcore",
                "/proc/keys",
                "/proc/latency_stats",
                "/proc/timer_list",
                "/proc/timer_stats",
                "/proc/sched_debug",
                "/proc/scsi",
                "/sys/firmware"
            ],
            "ReadonlyPaths": [
                "/proc/bus",
                "/proc/fs",
                "/proc/irq",
                "/proc/sys",
                "/proc/sysrq-trigger"
            ]
        },
        "GraphDriver": {
            "Data": {
                "LowerDir": "/var/lib/docker/overlay2/c935a891ecfe09bd8b5b3b923747adacc52bd29e5f42d3f42a61790a7677647c-init/diff:/var/lib/docker/overlay2/1782b79aa6d0329c2f4cb1882e2bea204c0118540b187b934277cad395b061c2/diff:/var/lib/docker/overlay2/aab0dadf085a30bd154795747ebd6ccfd0b2ec4f8c186b353ebda44dcf5df0e1/diff:/var/lib/docker/overlay2/e9973b879bada1914ca3f6593c9404db20f508c87fcf35721d0ec024dceb9ade/diff:/var/lib/docker/overlay2/6acecb8062f804cadc8962f8256b65b3f01ae6d81dcbe1caa7e381ec8f1709b3/diff",
                "MergedDir": "/var/lib/docker/overlay2/c935a891ecfe09bd8b5b3b923747adacc52bd29e5f42d3f42a61790a7677647c/merged",
                "UpperDir": "/var/lib/docker/overlay2/c935a891ecfe09bd8b5b3b923747adacc52bd29e5f42d3f42a61790a7677647c/diff",
                "WorkDir": "/var/lib/docker/overlay2/c935a891ecfe09bd8b5b3b923747adacc52bd29e5f42d3f42a61790a7677647c/work"
            },
            "Name": "overlay2"
        },
        "Mounts": [
            {
                "Type": "bind",
                "Source": "/var/run/docker.sock",
                "Destination": "/var/run/docker.sock",
                "Mode": "",
                "RW": false,
                "Propagation": "rprivate"
            },
            {
                "Type": "volume",
                "Name": "reverse-proxy_certificates",
                "Source": "/var/lib/docker/volumes/reverse-proxy_certificates/_data",
                "Destination": "/acme",
                "Driver": "local",
                "Mode": "z",
                "RW": true,
                "Propagation": ""
            }
        ],
        "Config": {
            "Hostname": "853176ebddfd",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "ExposedPorts": {
                "443/tcp": {},
                "5432/tcp": {},
                "80/tcp": {}
            },
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [
                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
            ],
            "Cmd": [
                "traefik"
            ],
            "Image": "traefik:v2.6@sha256:adf62c4c01290c29816f00b36a309df577771701e2feb2c9214fd3b6bd1e2894",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": [
                "/entrypoint.sh"
            ],
            "OnBuild": null,
            "Labels": {
                "com.docker.stack.namespace": "reverse-proxy",
                "com.docker.swarm.node.id": "lbwupro0h6kjs34p7gy2603m0",
                "com.docker.swarm.service.id": "w5az89qy1m7mvtwkm6jrzox5o",
                "com.docker.swarm.service.name": "reverse-proxy_reverse-proxy",
                "com.docker.swarm.task": "",
                "com.docker.swarm.task.id": "n8dxvft590laao30f7y1zspwe",
                "com.docker.swarm.task.name": "reverse-proxy_reverse-proxy.1.n8dxvft590laao30f7y1zspwe",
                "org.opencontainers.image.description": "A modern reverse-proxy",
                "org.opencontainers.image.documentation": "https://docs.traefik.io",
                "org.opencontainers.image.title": "Traefik",
                "org.opencontainers.image.url": "https://traefik.io",
                "org.opencontainers.image.vendor": "Traefik Labs",
                "org.opencontainers.image.version": "v2.6.7"
            }
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "904f1f845cd275175af86830c77ef9f917bdf45953be14175b897b1e5289c58b",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {
                "443/tcp": [
                    {
                        "HostIp": "0.0.0.0",
                        "HostPort": "443"
                    },
                    {
                        "HostIp": "::",
                        "HostPort": "443"
                    }
                ],
                "5432/tcp": [
                    {
                        "HostIp": "0.0.0.0",
                        "HostPort": "5432"
                    },
                    {
                        "HostIp": "::",
                        "HostPort": "5432"
                    }
                ],
                "80/tcp": [
                    {
                        "HostIp": "0.0.0.0",
                        "HostPort": "80"
                    },
                    {
                        "HostIp": "::",
                        "HostPort": "80"
                    }
                ]
            },
            "SandboxKey": "/var/run/docker/netns/904f1f845cd2",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "traefik_traefik-network": {
                    "IPAMConfig": {
                        "IPv4Address": "10.0.2.4"
                    },
                    "Links": null,
                    "Aliases": [
                        "853176ebddfd"
                    ],
                    "NetworkID": "gre6shjq24dnp3lb20hajeyxl",
                    "EndpointID": "532d9f9cc1d1127d4e03d5052054c7b96f8131c6b74e5d748bdf69323a7cacd5",
                    "Gateway": "",
                    "IPAddress": "10.0.2.4",
                    "IPPrefixLen": 24,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:0a:00:02:04",
                    "DriverOpts": null
                }
            }
        }
    }
]

As it's visible there is no Gateway assigned inside of the traefik container. Every other container regardless of node role also has none.

I already

The nodes are connected through an external IP address

Can anybody please tell me how to work around this issue?

neersighted commented 1 year ago

We're not sure if this is a presentation or substantial issue, though it sounds like it's just a 'value is not in the API' issue; likewise Gateway seems to have always been flaky: https://github.com/moby/moby/issues/26799

@akerouanton mentioned that he will look at this (and testing with Stack, specifically) as part of his network test suite work in the future.