search

allegro / marathon-consul

Integrates Marathon apps with Consul service discovery.
Apache License 2.0
191 stars 33 forks source link

USER network mode can't register to consul #284

Open nhanct opened 6 years ago

nhanct commented 6 years ago

I'm using USER mode network with calico and try to run mesos-consul but I see the logs:

INFO[0218] Syncing services finished. Stats, registerd: 1 (failed: 0), deregister: 0 (failed: 0). but can't find the service on consul

HOST network mode is ok

janisz commented 6 years ago

Can you show more logs? It should contain information what is registered. There is a chance healtchceck is not passing so service is not available.

nhanct commented 6 years ago

hi, here is log when i start an application

DEBU[0126] Asking Marathon for apps                      Location="marathon:8080"
DEBU[0126] Sending GET request to marathon               Location="marathon:8080" Protocol=http Uri="/v2/apps?embed=apps.tasks&label=consul"
DEBU[0126] Task is not healthy. Not Registering          ExpectedRegistrations=1 HasRegistrations=0 Id=redis.72c8ea4a-1bcf-11e8-a167-024222acc227 Sync=true
INFO[0126] Syncing services finished. Stats, registerd: 0 (failed: 0), deregister: 0 (failed: 0).
DEBU[0126] Leader detection disable
INFO[0126] Got HealthStatusEvent                         Id=redis.72c8ea4a-1bcf-11e8-a167-024222acc227
DEBU[0126] Asking Marathon for /redis                    Location="marathon:8080"
DEBU[0126] Sending GET request to marathon               Location="marathon:8080" Protocol=http Uri="/v2/apps//redis?embed=apps.tasks"
DEBU[0127] Forcing sync
INFO[0127] Syncing services started
DEBU[0127] Asking Marathon for apps                      Location="marathon:8080"
DEBU[0127] Sending GET request to marathon               Location="marathon:8080" Protocol=http Uri="/v2/apps?embed=apps.tasks&label=consul"
INFO[0127] Syncing services finished. Stats, registerd: 1 (failed: 0), deregister: 0 (failed: 0).
DEBU[0128] Forcing sync
INFO[0128] Syncing services started
DEBU[0128] Asking Marathon for apps                      Location="marathon:8080"
DEBU[0128] Sending GET request to marathon               Location="marathon:8080" Protocol=http Uri="/v2/apps?embed=apps.tasks&label=consul"
INFO[0128] Syncing services finished. Stats, registerd: 1 (failed: 0), deregister: 0 (failed: 0).
DEBU[0129] Forcing sync

the app pass health check but i can't see it on consul

There is some info about app:

    "Env": [
                "HOST=marathon",
                "MARATHON_APP_DOCKER_IMAGE=redis",
                "MARATHON_APP_ID=/redis",
                "MARATHON_APP_LABELS=CONSUL",
                "MARATHON_APP_LABEL_CONSUL=",
                "MARATHON_APP_RESOURCE_CPUS=1.0",
                "MARATHON_APP_RESOURCE_DISK=0.0",
                "MARATHON_APP_RESOURCE_GPUS=0",
                "MARATHON_APP_RESOURCE_MEM=100.0",
                "MARATHON_APP_VERSION=2018-02-27T15:04:03.012Z",
                "MESOS_CONTAINER_NAME=mesos-a89b8766-a1e5-47fd-9444-f9482545df1c",
                "MESOS_SANDBOX=/mnt/mesos/sandbox",
                "MESOS_TASK_ID=redis.72c8ea4a-1bcf-11e8-a167-024222acc227",
                "PORT0=6379",
                "PORTS=",
                "PORT_6379=6379",
                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
                "GOSU_VERSION=1.10",
                "REDIS_VERSION=4.0.8",
                "REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-4.0.8.tar.gz",
                "REDIS_DOWNLOAD_SHA=ff0c38b8c156319249fec61e5018cf5b5fe63a65b61690bec798f4c998c232ad"
            ],
            "Cmd": [
                "redis-server"
            ],
            "ArgsEscaped": true,
            "Image": "redis",
            "Volumes": {
                "/data": {}
            },
            "WorkingDir": "/data",
            "Entrypoint": [
                "docker-entrypoint.sh"
            ],
            "OnBuild": null,
            "Labels": {
                "MESOS_TASK_ID": "redis.72c8ea4a-1bcf-11e8-a167-024222acc227"
            }
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "33678def4b9cbec6f80b71edf1cf788d21f7c62bd26769d8356630041c21b5b2",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {},
            "SandboxKey": "/var/run/docker/netns/33678def4b9c",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "net1": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": [
                        "73fee30f8c88"
                    ],
                    "NetworkID": "62cac9c5ea1a3b0085527b937a7ab00fced17fe35aa0476dde743c973d2be69b",
                    "EndpointID": "975c656aeb81b4210e77464727d15708ac0ac31a13d15d5a106de1556bb084f8",
                    "Gateway": "169.254.1.1",
                    "IPAddress": "192.168.61.44",
                    "IPPrefixLen": 32,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "ee:ee:ee:ee:ee:ee"
                }
            }
        }
    }
]

I tested with HOST network mode and see it works as expected.

janisz commented 6 years ago

Could you post Minimal, Complete, and Verifiable example of marathon application JSON so I can test it? What versions are you using (Marathon, Consul, Marathon-Consul)?

zhanglt commented 6 years ago
Software Version
marathon 1.6.335
Consul 1.07
Marathon-Consul 1.4.2
Docker 17.05.0-ce 
[root@node1 ~]# docker network ls 
NETWORK ID          NAME                DRIVER              SCOPE
a96e392429fe        bridge                bridge                 local
ff4464cb6052        host                    host                    local
c512d25f75c4        my_net              calico                  global
df9ccac92157        none                  null                     local

marathon application JSON

{
  "id": "/nginx-net",
  "cmd": null,
  "cpus": 0.1,
  "mem": 64,
  "disk": 0,
  "instances": 3,
  "constraints": [
    [
      "hostname",
      "UNIQUE"
    ]
  ],
  "acceptedResourceRoles": [
    "*"
  ],
  "container": {
    "type": "DOCKER",
    "docker": {
      "forcePullImage": false,
      "image": "nginx",
      "parameters": [],
      "privileged": false
    },
    "volumes": [],
    "portMappings": [
      {
        "containerPort": 80,
        "labels": {},
        "protocol": "tcp",
        "servicePort": 10004
      }
    ]
  },
  "healthChecks": [
    {
      "gracePeriodSeconds": 300,
      "ignoreHttp1xx": false,
      "intervalSeconds": 60,
      "maxConsecutiveFailures": 3,
      "path": "/",
      "portIndex": 0,
      "protocol": "HTTP",
      "ipProtocol": "IPv4",
      "timeoutSeconds": 20,
      "delaySeconds": 15
    }
  ],
  "labels": {
    "consul": "nginx-net",
    "HAPROXY_GROUP": "proxy",
    "HAPROXY_0_VHOST": "node5.test"
  },
  "networks": [
    {
      "name": "my_net",
      "mode": "container"
    }
  ],
  "portDefinitions": []
}
[root@node1 ~]# docker network inspect my_net
[
    {
        "Name": "my_net",
        "Id": "c512d25f75c4e93b579559d1c39d08f85c39956916d900b62b8700633d81a472",
        "Created": "2018-04-21T23:13:42.947360347+08:00",
        "Scope": "global",
        "Driver": "calico",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "calico-ipam",
            "Options": {},
            "Config": [
                {
                    "Subnet": "192.0.2.0/24"
                }
            ]
        },
        "Internal": false,
        "Attachable": false,
        "Ingress": false,
        "Containers": {
            "3835c3c37b64cf4688fce64d6abb42587335aed72894da56dacf0e13067f9440": {
                "Name": "mesos-88ab805c-4a85-44c0-bd2f-c6826be5413e",
                "EndpointID": "533ea9f8685bbf75c9cb4aa08fa5603b3733b0474e36607d52076537052bcbdc",
                "MacAddress": "ee:ee:ee:ee:ee:ee",
                "IPv4Address": "192.0.2.68/32",
                "IPv6Address": ""
            },
            "ep-f50a38d45064c24d1a21ea469eb99e7fd1df7114274844c85999609f78fce782": {
                "Name": "mesos-1ca57deb-3502-483f-aa67-3e95ce41dbcd",
                "EndpointID": "f50a38d45064c24d1a21ea469eb99e7fd1df7114274844c85999609f78fce782",
                "MacAddress": "ee:ee:ee:ee:ee:ee",
                "IPv4Address": "192.0.2.7/32",
                "IPv6Address": ""
            },
            "ep-f5b2b0adfcdc7cc2b1b376ae67dad9d4509ffc715775862f10008363bd692bd4": {
                "Name": "mesos-fb3ee9b9-7b1a-4802-8f96-1dbcf5603a9e",
                "EndpointID": "f5b2b0adfcdc7cc2b1b376ae67dad9d4509ffc715775862f10008363bd692bd4",
                "MacAddress": "ee:ee:ee:ee:ee:ee",
                "IPv4Address": "192.0.2.199/32",
                "IPv6Address": ""
            }
        },
        "Options": {},
        "Labels": {}
    }
]