search

docker-archive / for-aws

92 stars 26 forks source link

Ingress? issue #205

Closed westfood closed 4 years ago

westfood commented 4 years ago

Expected behavior

Port is being exposed via ingress.

Actual behavior

Port is not being exposed via ingress. But only when Jenkins runs docker stack deploy, it's OK when running from my Macbook. Deployed stack file is always same. Both deployments goes thru same manager node.

Information

Steps to reproduce the behavior

version: "3.4"

networks:
 prometheus:
   external:
     name: swarm_prometheus

services:
  dispatcher:
    image: angelcam/recording-dispatcher-v2:{{ GIT_COMMIT }}
    hostname: rec-dispatcher-v2-{{ APP_ENV }}
    ports:
      - 8014:8088
    deploy:
      replicas: 1
      placement:
        constraints: [node.role == worker]
      labels:
        com.angelcam.name: rec-dispatcher-v2-{{ APP_ENV }}
        com.angelcam.env: swarm-{{ APP_ENV }}
    networks:
      - prometheus
    environment:
      LOGGLY_TOKEN: {{ LOGGLY_TOKEN }}
      LOGGLY_TAG: recording-dispatcher-{{ APP_ENV }}
      LOG_LEVEL: {{ LOG_LEVEL }}
      LOGSPOUT: ignore
      SENTRY_URL: {{ SENTRY_URL }}
      SENTRY_ENV: {{ APP_ENV }}
      DISPATCHER_API_SECRET: {{ DISPATCHER_API_SECRET }}
      MYAC_RECORDINGS_API_BASE_URL: {{ MYAC_RECORDINGS_API_BASE_URL }}
      MYAC_RECORDINGS_API_SECRET: {{ MYAC_RECORDINGS_API_SECRET }}
      AWS_ACCESS_KEY_ID: {{ AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: {{ AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: {{ AWS_REGION }}
      SNS_TOPIC_RECORDING_STATUS_CHANGED: {{ SNS_TOPIC_RECORDING_STATUS_CHANGED }}

My macbook:

Client: Docker Engine - Community
 Version:           19.03.4
 API version:       1.40
 Go version:        go1.12.10
 Git commit:        9013bf5
 Built:             Thu Oct 17 23:44:48 2019
 OS/Arch:           darwin/amd64
 Experimental:      false

Server: Docker Engine - Community
 Engine:
  Version:          19.03.4
  API version:      1.40 (minimum version 1.12)
  Go version:       go1.12.10
  Git commit:       9013bf5
  Built:            Thu Oct 17 23:50:38 2019
  OS/Arch:          linux/amd64
  Experimental:     true
 containerd:
  Version:          v1.2.10
  GitCommit:        b34a5c8af56e510852c35414db4c1f4fa6172339
 runc:
  Version:          1.0.0-rc8+dev
  GitCommit:        3e425f80a8c931f88e6d94a8c831b9d5aa481657
 docker-init:
  Version:          0.18.0

Jenkins

Client: Docker Engine - Community
 Version:           19.03.4
 API version:       1.40
 Go version:        go1.12.10
 Git commit:        9013bf583a
 Built:             Fri Oct 18 15:53:51 2019
 OS/Arch:           linux/amd64
 Experimental:      false

Server: Docker Engine - Community
 Engine:
  Version:          19.03.4
  API version:      1.40 (minimum version 1.12)
  Go version:       go1.12.10
  Git commit:       9013bf583a
  Built:            Fri Oct 18 15:52:23 2019
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          1.2.10
  GitCommit:        b34a5c8af56e510852c35414db4c1f4fa6172339
 runc:
  Version:          1.0.0-rc8+dev
  GitCommit:        3e425f80a8c931f88e6d94a8c831b9d5aa481657
 docker-init:
  Version:          0.18.0
  GitCommit:        fec3683

Stack deploy part of script

...
SWARM="docker -H localhost:${SSH_TUNNEL_PORT}"
...
      deploy_stack() {
        ${SWARM} stack deploy --with-registry-auth --prune -c ${TARGET_ENV}-recording.yaml recording \
        && echo "Stack deployed to Swarm test"
      }

docker service inspect is same for both deployments..

Resources:
Networks: swarm_prometheus
Endpoint Mode:  vip
Ports:
 PublishedPort = 8014
  Protocol = tcp
  TargetPort = 8088
  PublishMode = ingress
Resources:
Networks: swarm_prometheus
Endpoint Mode:  vip
Ports:
 PublishedPort = 8014
  Protocol = tcp
  TargetPort = 8088
  PublishMode = ingress

Swarm manager host's docker.log excerpt via Jenkins deployment..

Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.052228562Z" level=debug msg="Calling GET /v1.25/info"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.058277154Z" level=debug msg="Calling GET /v1.35/tasks?filters=%7B%22_up-to-date%22%3A%7B%22true%22%3Atrue%7D%2C%22service%22%3A%7B%22recording_dispatcher%22%3Atrue%7D%7D"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.067559341Z" level=debug msg="Calling GET /v1.25/nodes/lz3rbt9aevvp4hrsdz6qn5sfi"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.102827326Z" level=debug msg="Calling GET /v1.35/nodes"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.384117026Z" level=debug msg="Calling GET /v1.35/services/recording_dispatcher?insertDefaults=false"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.425992192Z" level=debug msg="Calling GET /v1.35/tasks?filters=%7B%22_up-to-date%22%3A%7B%22true%22%3Atrue%7D%2C%22service%22%3A%7B%22recording_dispatcher%22%3Atrue%7D%7D"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.469600561Z" level=debug msg="Calling GET /v1.35/nodes"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.643926938Z" level=debug msg="handleEpTableEvent ADD 37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 R:{recording_dispatcher.1.pw8gc0d8ncly6q9zarl5zockb recording_dispatcher q74dt3zq5fb2xikz8sc6ysoba 10.0.3.43 10.0.3.145 [] [dispatcher] [c08c32953608]}"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.644086862Z" level=debug msg="addServiceBinding from handleEpTableEvent START for recording_dispatcher 37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 p:0xc42533c780 nid:i8018a8xz3sm70q4huo3yslio skey:{q74dt3zq5fb2xikz8sc6ysoba }"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.644263862Z" level=debug msg="Creating service for vip 10.0.3.43 fwMark 700 ingressPorts []*libnetwork.PortConfig(nil) in sbox 7f4a182 (c35d0ba)"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42Z" level=warning msg="Running modprobe nf_nat failed with message: `modprobe: module nf_nat not found in modules.dep`, error: exit status 1"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42Z" level=warning msg="Running modprobe xt_conntrack failed with message: `modprobe: module xt_conntrack not found in modules.dep`, error: exit status 1"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.665911800Z" level=debug msg="Creating service for vip 10.0.3.43 fwMark 700 ingressPorts []*libnetwork.PortConfig(nil) in sbox 6fe29d3 (08a4269)"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42Z" level=warning msg="Running modprobe nf_nat failed with message: `modprobe: module nf_nat not found in modules.dep`, error: exit status 1"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42Z" level=warning msg="Running modprobe xt_conntrack failed with message: `modprobe: module xt_conntrack not found in modules.dep`, error: exit status 1"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.683953527Z" level=debug msg="addEndpointNameResolution 37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 recording_dispatcher add_service:true sAliases:[dispatcher] tAliases:[c08c32953608]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.683980168Z" level=debug msg="addContainerNameResolution 37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 recording_dispatcher.1.pw8gc0d8ncly6q9zarl5zockb"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.683994644Z" level=debug msg="37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 (i8018a8).addSvcRecords(recording_dispatcher.1.pw8gc0d8ncly6q9zarl5zockb, 10.0.3.145, <nil>, true) addServiceBinding sid:37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.684015782Z" level=debug msg="37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 (i8018a8).addSvcRecords(c08c32953608, 10.0.3.145, <nil>, true) addServiceBinding sid:37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.684033619Z" level=debug msg="37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 (i8018a8).addSvcRecords(tasks.recording_dispatcher, 10.0.3.145, <nil>, false) addServiceBinding sid:q74dt3zq5fb2xikz8sc6ysoba"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.684047002Z" level=debug msg="37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 (i8018a8).addSvcRecords(tasks.dispatcher, 10.0.3.145, <nil>, false) addServiceBinding sid:q74dt3zq5fb2xikz8sc6ysoba"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.684058340Z" level=debug msg="37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 (i8018a8).addSvcRecords(recording_dispatcher, 10.0.3.43, <nil>, false) addServiceBinding sid:q74dt3zq5fb2xikz8sc6ysoba"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.684069420Z" level=debug msg="37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30 (i8018a8).addSvcRecords(dispatcher, 10.0.3.43, <nil>, false) addServiceBinding sid:q74dt3zq5fb2xikz8sc6ysoba"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.684079489Z" level=debug msg="addServiceBinding from handleEpTableEvent END for recording_dispatcher 37f65e0ba56dd9931f099a233dc2fb0d54c7fe8ec3dbde2cc8c40fe2eaba7f30"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.752106018Z" level=debug msg="Calling GET /v1.35/services/recording_dispatcher?insertDefaults=false"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.762760754Z" level=debug msg="handleEpTableEvent ADD ba5125bdda781c32d72ce986cd595856384c744eecc29b3df862ae3e091c1d60 R:{recording_dispatcher.1.pw8gc0d8ncly6q9zarl5zockb recording_dispatcher q74dt3zq5fb2xikz8sc6ysoba 10.255.0.126 10.255.0.48 [&PortConfig{Name:,Protocol:TCP,TargetPort:8088,PublishedPort:8014,}]
[] [c08c32953608]}"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.762881372Z" level=debug msg="addServiceBinding from handleEpTableEvent START for recording_dispatcher ba5125bdda781c32d72ce986cd595856384c744eecc29b3df862ae3e091c1d60 p:0xc42533d000 nid:vkqc53ah8bvsjsvm3izn9kd96 skey:{q74dt3zq5fb2xikz8sc6ysoba 8014:8088/TCP}"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.763054261Z" level=debug msg="/sbin/iptables, [--wait -t nat -nL DOCKER-INGRESS]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.764749649Z" level=debug msg="/sbin/iptables, [--wait -t filter -nL DOCKER-INGRESS]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.765876279Z" level=debug msg="/sbin/iptables, [--wait -t nat -C DOCKER-INGRESS -j RETURN]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.766579119Z" level=debug msg="/sbin/iptables, [--wait -t filter -C DOCKER-INGRESS -j RETURN]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.767306548Z" level=debug msg="/sbin/iptables, [--wait -t nat -C OUTPUT -m addrtype --dst-type LOCAL -j DOCKER-INGRESS]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.768101665Z" level=debug msg="/sbin/iptables, [--wait -t nat -C PREROUTING -m addrtype --dst-type LOCAL -j DOCKER-INGRESS]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.768890171Z" level=debug msg="/sbin/iptables, [--wait -t filter -C FORWARD -j DOCKER-INGRESS]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.769728066Z" level=debug msg="/sbin/iptables, [--wait -t nat -C POSTROUTING -m addrtype --src-type LOCAL -o docker_gwbridge -j MASQUERADE]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.770645194Z" level=debug msg="/sbin/iptables, [--wait -t nat -nL DOCKER-INGRESS]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.771559596Z" level=debug msg="/sbin/iptables, [--wait -t nat -I DOCKER-INGRESS -p tcp --dport 8014 -j DNAT --to-destination 172.18.0.2:8014]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.772387730Z" level=debug msg="/sbin/iptables, [--wait -I DOCKER-INGRESS -m state -p tcp --sport 8014 --state ESTABLISHED,RELATED -j ACCEPT]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.773386368Z" level=debug msg="/sbin/iptables, [--wait -I DOCKER-INGRESS -p tcp --dport 8014 -j ACCEPT]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.774302814Z" level=debug msg="Creating service for vip 10.255.0.126 fwMark 701 ingressPorts []*libnetwork.PortConfig{&libnetwork.PortConfig{Name: \"\",\nProtocol: 0,\nTargetPort: 0x1f98,\nPublishedPort: 0x1f4e,\n}} in sbox ingress (ingress)"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42Z" level=warning msg="Running modprobe nf_nat failed with message: `modprobe: module nf_nat not found in modules.dep`, error: exit status 1"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42Z" level=warning msg="Running modprobe xt_conntrack failed with message: `modprobe: module xt_conntrack not found in modules.dep`, error: exit status 1"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.793761328Z" level=debug msg="Calling GET /v1.35/tasks?filters=%7B%22_up-to-date%22%3A%7B%22true%22%3Atrue%7D%2C%22service%22%3A%7B%22recording_dispatcher%22%3Atrue%7D%7D"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.795643287Z" level=debug msg="addEndpointNameResolution ba5125bdda781c32d72ce986cd595856384c744eecc29b3df862ae3e091c1d60 recording_dispatcher add_service:true sAliases:[] tAliases:[c08c32953608]"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.795671756Z" level=debug msg="addContainerNameResolution ba5125bdda781c32d72ce986cd595856384c744eecc29b3df862ae3e091c1d60 recording_dispatcher.1.pw8gc0d8ncly6q9zarl5zockb"
Nov 14 15:07:42 moby root: time="2019-11-14T15:07:42.795681510Z" level=debug msg="addServiceBinding from handleEpTableEvent END for recording_dispatcher ba5125bdda781c32d72ce986cd595856384c744eecc29b3df862ae3e091c1d60"

What else is strange is, when we deploy same image from Jenkins, this simple python process for app status

class StatusView(View):

    async def process(self, request):
        return Response(text='OK\n')

ends up with "curl: (56) Recv failure: Connection reset by peer"

root@rec-dispatcher-v2-test:/usr/src# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 127.0.0.11:40939        0.0.0.0:*               LISTEN      -
tcp        0      0 0.0.0.0:8088            0.0.0.0:*               LISTEN      5/python3
root@rec-dispatcher-v2-test:/usr/src# curl localhost:8088/status
curl: (56) Recv failure: Connection reset by peer

I guess there is something weird with networking in Swarm. We run ~20 services in swarm and this is first time, we have no clue, what is going on..

westfood commented 4 years ago

This error show up from time to time "Failed to deserialize netlink ndmsg: Link not found" not sure if it's connected to process of assigning address to endpoint


Nov 14 16:12:36 moby root: time="2019-11-14T16:12:36.624879002Z" level=debug msg="Assigning addresses for endpoint gateway_38e694eb976e's interface on network docker_gwbridge"
Nov 14 16:12:36 moby root: time="2019-11-14T16:12:36.624963732Z" level=debug msg="RequestAddress(LocalDefault/172.18.0.0/16, <nil>, map[])"
Nov 14 16:12:36 moby root: time="2019-11-14T16:12:36.627127818Z" level=error msg="Failed to deserialize netlink ndmsg: Link not found"```
westfood commented 4 years ago

And this one is from deployment from macbook whi h workss

14 16:29:55 moby root: time="2019-11-14T16:29:55.098958073Z" level=debug msg=event module=libcontainerd namespace=moby topic=/containers/delete
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.098987289Z" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/containers/delete type="*events.ContainerDelete"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.104338833Z" level=debug msg="state changed" module=node/agent/taskmanager node.id=m5l02512b1qtare3ee9ec4w02 service.id=q74dt3zq5fb2xikz8sc6ysoba state.desired=SHUTDOWN state.transition="RUNNING->SHUTDOWN" task.id=63q0kckcq25ksa1slvbjcqjrw
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.104777117Z" level=debug msg="(*Agent).UpdateTaskStatus" module=node/agent node.id=m5l02512b1qtare3ee9ec4w02 task.id=63q0kckcq25ksa1slvbjcqjrw
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.108130181Z" level=debug msg="task status reported" module=node/agent node.id=m5l02512b1qtare3ee9ec4w02
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.108158363Z" level=debug msg="(*Agent).UpdateTaskStatus" module=node/agent node.id=m5l02512b1qtare3ee9ec4w02 task.id=63q0kckcq25ksa1slvbjcqjrw
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.110895691Z" level=debug msg="task status reported" module=node/agent node.id=m5l02512b1qtare3ee9ec4w02
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.257032023Z" level=debug msg="(*worker).Update" len(assignments)=1 module=node/agent node.id=m5l02512b1qtare3ee9ec4w02
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.257077533Z" level=debug msg="(*worker).reconcileSecrets" len(removedSecrets)=0 len(updatedSecrets)=0 module=node/agent node.id=m5l02512b1qtare3ee9ec4w02
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.257094760Z" level=debug msg="(*worker).reconcileConfigs" len(removedConfigs)=0 len(updatedConfigs)=0 module=node/agent node.id=m5l02512b1qtare3ee9ec4w02
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.257108509Z" level=debug msg="(*worker).reconcileTaskState" len(removedTasks)=0 len(updatedTasks)=1 module=node/agent node.id=m5l02512b1qtare3ee9ec4w02
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.257132506Z" level=debug msg=assigned module=node/agent node.id=m5l02512b1qtare3ee9ec4w02 task.desiredstate=RUNNING task.id=x8h79ddgy8lg0df18uad345rk
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.257912621Z" level=debug msg="state changed" module=node/agent/taskmanager node.id=m5l02512b1qtare3ee9ec4w02 service.id=q74dt3zq5fb2xikz8sc6ysoba state.desired=RUNNING state.transition="READY->STARTING" task.id=x8h79ddgy8lg0df18uad345rk
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.258890705Z" level=debug msg="(*Agent).UpdateTaskStatus" module=node/agent node.id=m5l02512b1qtare3ee9ec4w02 task.id=x8h79ddgy8lg0df18uad345rk
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.261057294Z" level=debug msg="container mounted via layerStore: &{/var/lib/docker/overlay2/d77ccc20516b72144fcc988942a271e0fbbf192330339b826a9b39869ad4eb83/merged 0x2c40ee0 0x2c40ee0}"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.261473328Z" level=debug msg="Assigning addresses for endpoint recording_dispatcher.1.x8h79ddgy8lg0df18uad345rk's interface on network ingress"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.261504810Z" level=debug msg="RequestAddress(LocalDefault/10.255.0.0/16, 10.255.0.52, map[])"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.262512615Z" level=debug msg="task status reported" module=node/agent node.id=m5l02512b1qtare3ee9ec4w02
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.262851332Z" level=debug msg="Assigning addresses for endpoint recording_dispatcher.1.x8h79ddgy8lg0df18uad345rk's interface on network ingress"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.373942049Z" level=debug msg="checkEncryption(vkqc53a, <nil>, 4096, true)"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.374975514Z" level=error msg="Failed to deserialize netlink ndmsg: Link not found"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.374992058Z" level=debug msg="Assigning addresses for endpoint gateway_5d4febfda450's interface on network docker_gwbridge"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.375079838Z" level=debug msg="RequestAddress(LocalDefault/172.18.0.0/16, <nil>, map[])"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.378054328Z" level=debug msg="Assigning addresses for endpoint gateway_5d4febfda450's interface on network docker_gwbridge"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.382352760Z" level=debug msg="Programming external connectivity on endpoint gateway_5d4febfda450 (b0d5c9038d6305b6d53fae7fa01af368ac1a471ddcfb168474d425ea8573d92a)"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.385126006Z" level=debug msg="Assigning addresses for endpoint recording_dispatcher.1.x8h79ddgy8lg0df18uad345rk's interface on network swarm_prometheus"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.385155136Z" level=debug msg="RequestAddress(LocalDefault/10.0.3.0/24, 10.0.3.149, map[])"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.386552713Z" level=debug msg="Assigning addresses for endpoint recording_dispatcher.1.x8h79ddgy8lg0df18uad345rk's interface on network swarm_prometheus"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.523181384Z" level=error msg="Failed to deserialize netlink ndmsg: Link not found"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.524477502Z" level=debug msg="checkEncryption(i8018a8, <nil>, 4100, true)"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.528871169Z" level=debug msg="bundle dir created" bundle=/var/run/docker/containerd/5d4febfda450db158d5d36068ec50956851b33c334f8be3465006479d94d2fea module=libcontainerd namespace=moby root=/var/lib/docker/overlay2/d77ccc20516b72144fcc988942a271e0fbbf192330339b826a9b39869ad4eb83/merged
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55Z" level=debug msg="event published" module="containerd/containers" ns=moby topic="/containers/create" type=containerd.events.ContainerCreate
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.533233611Z" level=debug msg=event module=libcontainerd namespace=moby topic=/containers/create
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.533258648Z" level=info msg="ignoring event" module=libcontainerd namespace=moby topic=/containers/create type="*events.ContainerCreate"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55Z" level=info msg="shim docker-containerd-shim started" address="/containerd-shim/moby/5d4febfda450db158d5d36068ec50956851b33c334f8be3465006479d94d2fea/shim.sock" debug=true module="containerd/tasks" pid=19171
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55Z" level=debug msg="registering ttrpc server"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55Z" level=debug msg="serving api on unix socket" socket="[inherited from parent]"
Nov 14 16:29:55 moby root: time="2019-11-14T16:29:55.548158141Z" level=debug msg="Calling GET /containers/5d4febfda450db158d5d36068ec50956851b33c334f8be3465006479d94d2fea/json"
westfood commented 4 years ago

Ok, there has been mismatch between what image is under python:3 in Jenkins and my macbook. “PYTHON_VERSION=3.7.3”, “PYTHON_VERSION=3.8.0",