itzg
commented
1 year ago @n1xx1 do you have any thoughts on this one?
Closed laiskimus closed 1 year ago
itzg
commented
1 year ago @n1xx1 do you have any thoughts on this one?
n1xx1
commented
1 year ago While I was testing my network configuration was like this:
It looks really weird that you are getting the wrong IP. I would ask you to send us the result of docker inspect for both your 2 containers and 2 services.
laiskimus
commented
1 year ago @n1xx1 here are the inspects
docker network inspect minecraft_minecraft ingress
[
{
"Name": "minecraft_minecraft",
"Id": "i4bz09bukyfksemy78h4ehwap",
"Created": "2022-11-15T08:02:59.036055919Z",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.4.0/24",
"Gateway": "10.0.4.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"12bbcac3e235fd8f1eec9bc0cfa2296223c5dc01f1cf18e1079ddcdf66ef5e8a": {
"Name": "minecraft_riimumestari_s4.1.8k3iethaors3v9zbobjhl1p7x",
"EndpointID": "c62ba1c831279c8bbe6aa7591a49ff52dd9d090f6b22e84bc3a81fb3f8841f7b",
"MacAddress": "02:42:0a:00:04:05",
"IPv4Address": "10.0.4.5/24",
"IPv6Address": ""
},
"1786f2588f4bd43d6bea4ed5120d0b556fd096a9cdc7f56e2c29c189191de0e4": {
"Name": "minecraft_router.1.4ne4vuy6364xoqb81x5a82hqf",
"EndpointID": "1cf7462bbaa0a286c205d86789ec5e12ae1d636e03e06946c33004d691fd2829",
"MacAddress": "02:42:0a:00:04:07",
"IPv4Address": "10.0.4.7/24",
"IPv6Address": ""
},
"lb-minecraft_minecraft": {
"Name": "minecraft_minecraft-endpoint",
"EndpointID": "fed234f371bac47759cc8ede9917b2c1b7b8ffb80e11ab3147500b548ed7c340",
"MacAddress": "02:42:0a:00:04:04",
"IPv4Address": "10.0.4.4/24",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4100"
},
"Labels": {
"com.docker.stack.namespace": "minecraft"
},
"Peers": [
{
"Name": "6382bea472a0",
"IP": "10.10.240.14"
}
]
},
{
"Name": "ingress",
"Id": "tfkafnkvlgtovx3cg8xt1uvjk",
"Created": "2022-11-15T08:02:58.714924706Z",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.0.0/24",
"Gateway": "10.0.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": true,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"12bbcac3e235fd8f1eec9bc0cfa2296223c5dc01f1cf18e1079ddcdf66ef5e8a": {
"Name": "minecraft_riimumestari_s4.1.8k3iethaors3v9zbobjhl1p7x",
"EndpointID": "2094f20ed167d5ce20a6bd573b441ddd0b9d422d02325dd005b90c01b70ebddc",
"MacAddress": "02:42:0a:00:00:09",
"IPv4Address": "10.0.0.9/24",
"IPv6Address": ""
},
"1786f2588f4bd43d6bea4ed5120d0b556fd096a9cdc7f56e2c29c189191de0e4": {
"Name": "minecraft_router.1.4ne4vuy6364xoqb81x5a82hqf",
"EndpointID": "9f57148ad9434644736312ed144e64ebb5961b249db74567e9fda769eccba4a7",
"MacAddress": "02:42:0a:00:00:0b",
"IPv4Address": "10.0.0.11/24",
"IPv6Address": ""
},
"ingress-sbox": {
"Name": "ingress-endpoint",
"EndpointID": "77c6d1d373f93df4378389f9d586388c38dcc54043fc2fa7a63b27bca50163fa",
"MacAddress": "02:42:0a:00:00:02",
"IPv4Address": "10.0.0.2/24",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4096"
},
"Labels": {},
"Peers": [
{
"Name": "6382bea472a0",
"IP": "10.10.240.14"
}
]
}
]docker service inspect minecraft_router minecraft_riimumestari_s4
[
{
"ID": "lilns3455oy384isa6dfyimww",
"Version": {
"Index": 5518
},
"CreatedAt": "2022-11-15T10:36:38.38795593Z",
"UpdatedAt": "2022-11-15T10:36:38.392457554Z",
"Spec": {
"Name": "minecraft_router",
"Labels": {
"com.docker.stack.image": "itzg/mc-router:latest",
"com.docker.stack.namespace": "minecraft"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "itzg/mc-router:latest@sha256:c9aecc52a800c4aee0c2eb7fb8f610e85caa2334ca21a0e312ac8c8e095d4f49",
"Labels": {
"com.docker.stack.namespace": "minecraft"
},
"Args": [
"--in-docker-swarm",
"--debug"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/run/docker.sock",
"Target": "/var/run/docker.sock"
}
],
"StopGracePeriod": 10000000000,
"DNSConfig": {},
"Isolation": "default"
},
"Resources": {},
"RestartPolicy": {
"Condition": "any",
"Delay": 5000000000,
"MaxAttempts": 0
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
},
{
"OS": "linux"
},
{
"Architecture": "arm64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "i4bz09bukyfksemy78h4ehwap",
"Aliases": [
"router"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"UpdateConfig": {
"Parallelism": 1,
"FailureAction": "pause",
"Monitor": 5000000000,
"MaxFailureRatio": 0,
"Order": "stop-first"
},
"RollbackConfig": {
"Parallelism": 1,
"FailureAction": "pause",
"Monitor": 5000000000,
"MaxFailureRatio": 0,
"Order": "stop-first"
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 25565,
"PublishedPort": 25565,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 25565,
"PublishedPort": 25565,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 25565,
"PublishedPort": 25565,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "tfkafnkvlgtovx3cg8xt1uvjk",
"Addr": "10.0.0.10/24"
},
{
"NetworkID": "i4bz09bukyfksemy78h4ehwap",
"Addr": "10.0.4.6/24"
}
]
}
},
{
"ID": "y8m80eu5uj4zdyyohv135tahv",
"Version": {
"Index": 5509
},
"CreatedAt": "2022-11-14T18:16:16.599974141Z",
"UpdatedAt": "2022-11-15T10:35:00.636358382Z",
"Spec": {
"Name": "minecraft_riimumestari_s4",
"Labels": {
"com.docker.stack.image": "itzg/minecraft-server:java17-alpine",
"com.docker.stack.namespace": "minecraft",
"mc-router.host": "season4.mc.riimumestari.net"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "itzg/minecraft-server:java17-alpine@sha256:a648904aa0818ef8b0b21f6b83fc4111153b272f2a2a133f9d02365c6cb39649",
"Labels": {
"com.docker.stack.namespace": "minecraft"
},
"Hostname": "riimumestari_r4",
"Env": [
"DIFFICULTY=hard",
"EULA=TRUE",
"MODE=survival",
"REMOVE_OLD_MODS=TRUE",
"REMOVE_OLD_VANILLATWEAKS=TRUE",
"TYPE=FABRIC",
"TZ=Europe/Helsinki",
"VANILLATWEAKS_FILE=/vanillatweaks-datapacks.json",
"VERSION=1.19.2"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"TTY": true,
"OpenStdin": true,
"Mounts": [
{
"Type": "volume",
"Source": "minecraft_riimumestari_s4_data",
"Target": "/data",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "minecraft"
},
"DriverConfig": {
"Name": "local",
"Options": {
"device": "/opt/docker/minecraft/riimumestari_s4/data",
"o": "bind",
"type": "none"
}
}
}
},
{
"Type": "volume",
"Source": "minecraft_riimumestari_s4_mods",
"Target": "/mods",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "minecraft"
},
"DriverConfig": {
"Name": "local",
"Options": {
"device": "/opt/docker/minecraft/riimumestari_s4/mods",
"o": "bind",
"type": "none"
}
}
}
}
],
"StopGracePeriod": 10000000000,
"DNSConfig": {},
"Configs": [
{
"File": {
"Name": "/vanillatweaks-datapacks.json",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "ittol3okgbd3v78c1jzs5is2r",
"ConfigName": "riimumestari_s4_vanillatweaks-datapacks-20221115-103402"
}
],
"Isolation": "default"
},
"Resources": {},
"RestartPolicy": {
"Condition": "any",
"Delay": 5000000000,
"MaxAttempts": 0
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "i4bz09bukyfksemy78h4ehwap",
"Aliases": [
"riimumestari_s4"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"UpdateConfig": {
"Parallelism": 1,
"FailureAction": "pause",
"Monitor": 5000000000,
"MaxFailureRatio": 0,
"Order": "stop-first"
},
"RollbackConfig": {
"Parallelism": 1,
"FailureAction": "pause",
"Monitor": 5000000000,
"MaxFailureRatio": 0,
"Order": "stop-first"
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 8123,
"PublishedPort": 8123,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 25565,
"PublishedPort": 25570,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 25575,
"PublishedPort": 25579,
"PublishMode": "ingress"
}
]
}
},
"PreviousSpec": {
"Name": "minecraft_riimumestari_s4",
"Labels": {
"com.docker.stack.image": "itzg/minecraft-server:java17-alpine",
"com.docker.stack.namespace": "minecraft",
"mc-router.host": "season4.mc.riimumestari.net"
},
"TaskTemplate": {
"ContainerSpec": {
"Image": "itzg/minecraft-server:java17-alpine@sha256:a648904aa0818ef8b0b21f6b83fc4111153b272f2a2a133f9d02365c6cb39649",
"Labels": {
"com.docker.stack.namespace": "minecraft"
},
"Hostname": "riimumestari_r4",
"Env": [
"DIFFICULTY=hard",
"EULA=TRUE",
"MODE=survival",
"REMOVE_OLD_MODS=TRUE",
"REMOVE_OLD_VANILLATWEAKS=TRUE",
"TYPE=FABRIC",
"TZ=Europe/Helsinki",
"VANILLATWEAKS_FILE=/vanillatweaks-datapacks.json",
"VERSION=1.19.2"
],
"Privileges": {
"CredentialSpec": null,
"SELinuxContext": null
},
"TTY": true,
"OpenStdin": true,
"Mounts": [
{
"Type": "volume",
"Source": "minecraft_riimumestari_s4_data",
"Target": "/data",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "minecraft"
},
"DriverConfig": {
"Name": "local",
"Options": {
"device": "/opt/docker/minecraft/riimumestari_s4/data",
"o": "bind",
"type": "none"
}
}
}
},
{
"Type": "volume",
"Source": "minecraft_riimumestari_s4_mods",
"Target": "/mods",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "minecraft"
},
"DriverConfig": {
"Name": "local",
"Options": {
"device": "/opt/docker/minecraft/riimumestari_s4/mods",
"o": "bind",
"type": "none"
}
}
}
}
],
"Configs": [
{
"File": {
"Name": "/vanillatweaks-datapacks.json",
"UID": "0",
"GID": "0",
"Mode": 292
},
"ConfigID": "8wiyhw6fbo31zcb98f7u2qnm4",
"ConfigName": "riimumestari_s4_vanillatweaks-datapacks-20221114-181614"
}
],
"Isolation": "default"
},
"Resources": {},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"Networks": [
{
"Target": "i4bz09bukyfksemy78h4ehwap",
"Aliases": [
"riimumestari_s4"
]
}
],
"ForceUpdate": 0,
"Runtime": "container"
},
"Mode": {
"Replicated": {
"Replicas": 1
}
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 8123,
"PublishedPort": 8123,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 25565,
"PublishedPort": 25570,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 25575,
"PublishedPort": 25579,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 8123,
"PublishedPort": 8123,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 25565,
"PublishedPort": 25570,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 25575,
"PublishedPort": 25579,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 8123,
"PublishedPort": 8123,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 25565,
"PublishedPort": 25570,
"PublishMode": "ingress"
},
{
"Protocol": "tcp",
"TargetPort": 25575,
"PublishedPort": 25579,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "tfkafnkvlgtovx3cg8xt1uvjk",
"Addr": "10.0.0.56/24"
},
{
"NetworkID": "i4bz09bukyfksemy78h4ehwap",
"Addr": "10.0.4.16/24"
}
]
},
"UpdateStatus": {
"State": "completed",
"StartedAt": "2022-11-15T10:34:04.756295603Z",
"CompletedAt": "2022-11-15T10:35:00.636339915Z",
"Message": "update completed"
}
}
]If you use the provided compose file it shouldn't add the containers to the ingress network. However maybe it would make sense to provide some configuration label to specify the network it should consider when finding the VirtualIPs. I may be able to put up a PR this week.
laiskimus
commented
1 year ago @n1xx1 thanks. I think the ingress network comes from defining ports
Specifying the network would be nice. I'm planning of running Dynamap on the servers and planning routing that to traefik on different network
n1xx1
commented
1 year ago I think the
ingressnetwork comes from definingports
You're supposed to specify the ports only in the router service. Can you share the compose file?
laiskimus
commented
1 year ago You're supposed to specify the ports only in the router service. Can you share the compose file?
Oh, ok. I understood that router did not need the ports in the minecraft service, I did not realize that it affects the router.
compose file below.
I tested with new setup. Looks like my problem was those ports in the minecraft server service.
when ports is defined minecraft server service:
when ports is NOT defined minecraft server service:
version: '3.8'
services:
router:
image: itzg/mc-router:latest
command: --in-docker-swarm --debug
deploy:
mode: replicated
replicas: 1
networks:
- minecraft
ports:
- "25565:25565"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
riimumestari_test:
image: itzg/minecraft-server:java17-alpine
hostname: riimumestari_test
tty: true
stdin_open: true
deploy:
mode: replicated
replicas: 1
labels:
- "mc-router.host=test.mc.riimumestari.net"
environment:
EULA: "TRUE"
VERSION: "1.19.2"
MODE: "survival"
DIFFICULTY: "hard"
PVP: "false"
MOTD: "Riimumestari test Server"
TYPE: "FABRIC"
REMOVE_OLD_MODS: "TRUE"
VANILLATWEAKS_FILE: /vanillatweaks-datapacks.json
REMOVE_OLD_VANILLATWEAKS: "TRUE"
TZ: "Europe/Helsinki"
networks:
- minecraft
ports:
- "8123:8123" # Dynamap
# - "7091:7091" # jmx
- "25570:25565" # Minecraft
- "25579:25575" # RCON
configs:
- source: riimumestari_test_vanillatweaks_datapacks
target: /vanillatweaks-datapacks.json
volumes:
- riimumestari_test_data:/data
- riimumestari_test_mods:/mods
configs:
riimumestari_test_vanillatweaks_datapacks:
name: "riimumestari_test_vanillatweaks-datapacks-${DEPLOYTIMEDATE-default}"
file: ./config/riimumestari_test_vanillatweaks-datapacks.json
volumes:
riimumestari_test_data:
driver: local
driver_opts:
type: none
o: bind
device: "/opt/docker/minecraft/riimumestari_test/data"
riimumestari_test_mods:
driver: local
driver_opts:
type: none
o: bind
device: "/opt/docker/minecraft/riimumestari_test/mods"
networks:
minecraft:
driver: overlayI've submitted a PR with the documentation on how to use the new feature. In the meanwhile you could avoid exposing ports so that docker doesn't attach the ingress network, or maybe you can setup something that does service discovery for TCP like traefik. However consider it's not the easiest thing to configure, so you may want to wait for the new release 😄
laiskimus
commented
1 year ago @n1xx1 thanks again, I'll be waiting 😄 While I wait I might try running whole thing in K8s
I tried deploy to docker swarm with provided samples. --in-docker-swarm
log from router container
Network configs
in my case, somehow router gets IP from "ingress" network not "minecraft"
also, tested starting minecraft server service with ports defined for accessing directly, backend IP on logs was from ingress ip range.
should there be some way to defined overlay networks name to use in router and servers ?