search

kubeovn / kube-ovn

A Bridge between SDN and Cloud Native (Project under CNCF)
https://kubeovn.github.io/docs/stable/en/
Apache License 2.0
1.91k stars 435 forks source link

ovn-default pod with eip create failed #3008

Closed bobz965 closed 1 year ago

bobz965 commented 1 year ago

Expected Behavior

ovn-default pod with eip create ok

Actual Behavior

ovn-default pod with eip create failed

Steps to Reproduce the Problem

  1. deploy master install.sh
  2. create provider network, apply configmap to enable eip snat
  3. create a pod with annos to use eip

Additional Info

# my env

root@empty:~/test/kube-ovn# k get subnet
NAME           PROVIDER   VPC           PROTOCOL   CIDR             PRIVATE   NAT     DEFAULT   GATEWAYTYPE   V4USED   V4AVAILABLE   V6USED   V6AVAILABLE   EXCLUDEIPS        U2OINTERCONNECTIONIP
external       ovn        ovn-cluster   IPv4       192.168.7.0/24   false     false   false     distributed   0        253           0        0             ["192.168.7.1"]
join           ovn        ovn-cluster   IPv4       100.64.0.0/16    false     false   false     distributed   1        65532         0        0             ["100.64.0.1"]
ovn-default    ovn        ovn-cluster   IPv4       10.16.0.0/16     false     true    true      distributed   9        65524         0        0             ["10.16.0.1"]
vpc1-subnet1   ovn        vpc1          IPv4       10.1.0.0/24      false     false   false     distributed   2        251           0        0             ["10.1.0.1"]
vpc2-subnet1   ovn        vpc2          IPv4       10.2.0.0/24      false     false   false     distributed   2        251           0        0             ["10.2.0.1"]
vpc3-subnet1   ovn        vpc3          IPv4       10.3.0.0/24      false     false   false     distributed   2        251           0        0             ["10.3.0.1"]
root@empty:~/test/kube-ovn# k ko nbctl show
switch 97edb091-b2b0-4e90-9f82-43ec7b7f0546 (join)
    port node-empty
        addresses: ["00:00:00:41:A4:1B 100.64.0.2"]
    port join-ovn-cluster
        type: router
        router-port: ovn-cluster-join
switch 4a0e1a05-9ff5-4dd4-a7f4-07501a347ff0 (vpc1-subnet1)
    port vpc1-subnet1-vpc1
        type: router
        router-port: vpc1-vpc1-subnet1
    port moon-0.ns1
        addresses: ["00:00:00:D4:1C:94 10.1.0.11"]
    port alice-557c6f9688-pr48r.ns1
        addresses: ["00:00:00:F6:35:C1 10.1.0.2"]
switch 5a8fc872-7236-4096-8b37-53be5ace76b9 (vpc3-subnet1)
    port vpc3-subnet1-vpc3
        type: router
        router-port: vpc3-vpc3-subnet1
    port mars-0.ns1
        addresses: ["00:00:00:D6:CD:61 10.3.0.33"]
    port carrie-6c69f77d8d-682ms.ns1
        addresses: ["00:00:00:A6:68:AC 10.3.0.2"]
switch 320c63b3-9ddf-42c1-9d25-556eb234231f (ovn-default)
    port cert-manager-b79f6b7dd-d8dcn.cert-manager
        addresses: ["00:00:00:EE:01:8A 10.16.0.5"]
    port cert-manager-cainjector-788d4bd865-dttm8.cert-manager
        addresses: ["00:00:00:A4:B4:04 10.16.0.4"]
    port pod-with-eip.default
        addresses: ["00:00:00:B3:DB:42 10.16.0.9"]
    port coredns-757cd945b-t6p9x.kube-system
        addresses: ["00:00:00:A3:7A:41 10.16.0.3"]
    port cert-manager-webhook-767b7d59cf-zcgjr.cert-manager
        addresses: ["00:00:00:A1:09:EF 10.16.0.6"]
    port ovn-default-fip.default
        addresses: ["00:00:00:BD:13:7C 10.16.0.10"]
    port ovn-default-ovn-cluster
        type: router
        router-port: ovn-cluster-ovn-default
    port coredns-757cd945b-znss7.kube-system
        addresses: ["00:00:00:0E:51:F4 10.16.0.2"]
    port vpn-gw-controller-manager-6767d57475-vcxzw.vpn-gw-system
        addresses: ["00:00:00:C7:0E:0D 10.16.0.7"]
    port kube-ovn-pinger-7pszm.kube-system
        addresses: ["00:00:00:53:2D:36 10.16.0.8"]
switch 8fd2a7f7-8929-48fb-aa3d-500468e0df89 (external)
    port localnet.external
        type: localnet
        addresses: ["unknown"]
    port external-vpc3
        type: router
        router-port: vpc3-external
    port external-ovn-cluster
        type: router
        router-port: ovn-cluster-external
    port external-vpc2
        type: router
        router-port: vpc2-external
    port external-vpc1
        type: router
        router-port: vpc1-external
switch 1559f2c3-f7a5-4cd8-834a-2fe7c850e17b (vpc2-subnet1)
    port sun-0.ns1
        addresses: ["00:00:00:3D:3E:21 10.2.0.22"]
    port bob-86f595f954-lgpnp.ns1
        addresses: ["00:00:00:5C:C0:16 10.2.0.2"]
    port vpc2-subnet1-vpc2
        type: router
        router-port: vpc2-vpc2-subnet1
router 68a63468-00c3-48b9-a37b-1b3be0d0dbb7 (vpc1)
    port vpc1-external
        mac: "00:00:00:3A:E1:51"
        networks: ["192.168.7.2/24"]
        gateway chassis: [5173ad80-17da-4b13-8fa1-cd33d6024121]
    port vpc1-vpc1-subnet1
        mac: "00:00:00:78:5B:B5"
        networks: ["10.1.0.1/24"]
    nat 8f80b7cc-bf55-4997-8320-41e8caf187bf
        external ip: "192.168.7.11"
        logical ip: "10.1.0.11"
        type: "dnat_and_snat"
router e7f25991-e110-4afe-bce6-2f13208f1c13 (vpc3)
    port vpc3-external
        mac: "00:00:00:15:BC:20"
        networks: ["192.168.7.4/24"]
        gateway chassis: [5173ad80-17da-4b13-8fa1-cd33d6024121]
    port vpc3-vpc3-subnet1
        mac: "00:00:00:01:6C:A9"
        networks: ["10.3.0.1/24"]
    nat 8a61e4f9-2bd0-4cf9-97f8-5db0042c00c8
        external ip: "192.168.7.33"
        logical ip: "10.3.0.33"
        type: "dnat_and_snat"
router 52bc05d6-a1a3-4af8-abf0-101d46fee6a5 (ovn-cluster)
    port ovn-cluster-external
        mac: "00:00:00:B3:EC:47"
        networks: ["192.168.7.1/24"]
        gateway chassis: [5173ad80-17da-4b13-8fa1-cd33d6024121]
    port ovn-cluster-ovn-default
        mac: "00:00:00:F1:A0:9D"
        networks: ["10.16.0.1/16"]
    port ovn-cluster-join
        mac: "00:00:00:C9:AA:1A"
        networks: ["100.64.0.1/16"]
    nat ef4add75-5d58-4d41-8824-62e7f7652c27
        external ip: "192.168.7.77"
        logical ip: "10.16.0.10"
        type: "dnat_and_snat"
router 2022e5d7-912d-488c-946f-fda6d7995244 (vpc2)
    port vpc2-vpc2-subnet1
        mac: "00:00:00:16:0B:B6"
        networks: ["10.2.0.1/24"]
    port vpc2-external
        mac: "00:00:00:32:48:72"
        networks: ["192.168.7.3/24"]
        gateway chassis: [5173ad80-17da-4b13-8fa1-cd33d6024121]
    nat 747dc9f7-99c7-453f-9369-726ca29317d9
        external ip: "192.168.7.22"
        logical ip: "10.2.0.22"
        type: "dnat_and_snat"
root@empty:~/test/kube-ovn#

# create pod with eip
---
apiVersion: v1
kind: Pod
metadata:
  name: pod-with-eip
  annotations:
    ovn.kubernetes.io/eip: 192.168.7.177
spec:
  containers:
  - name: netshoot
    image: nicolaka/netshoot
    imagePullPolicy: Never
    command: ["/bin/bash"]
    args: ["-c", "while true; do ping localhost; sleep 60;done"]

# pod create err log

root@empty:~/test/kube-ovn# k describe po -n default         pod-with-eip
Name:         pod-with-eip
Namespace:    default
Priority:     0
Node:         empty/172.20.212.181
Start Time:   Mon, 03 Jul 2023 15:57:11 +0800
Labels:       <none>
Annotations:  ovn.kubernetes.io/allocated: true
              ovn.kubernetes.io/cidr: 10.16.0.0/16
              ovn.kubernetes.io/eip: 192.168.7.177
              ovn.kubernetes.io/gateway: 10.16.0.1
              ovn.kubernetes.io/ip_address: 10.16.0.9
              ovn.kubernetes.io/logical_router: ovn-cluster
              ovn.kubernetes.io/logical_switch: ovn-default
              ovn.kubernetes.io/mac_address: 00:00:00:B3:DB:42
              ovn.kubernetes.io/pod_nic_type: veth-pair
Status:       Pending
IP:
IPs:          <none>
Containers:
  netshoot:
    Container ID:
    Image:         nicolaka/netshoot
    Image ID:
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/bash
    Args:
      -c
      while true; do ping localhost; sleep 60;done
    State:          Waiting
      Reason:       ContainerCreating
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-2pvcw (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  kube-api-access-2pvcw:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason                  Age    From               Message
  ----     ------                  ----   ----               -------
  Normal   Scheduled               4m2s   default-scheduler  Successfully assigned default/pod-with-eip to empty
  Warning  FailedCreatePodSandBox  3m39s  kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "aec9ab6585531f87f599e2174b56f8c5e0c486a8edf1be3d7bcdda6b5045a63d" network for pod "pod-with-eip": networkPlugin cni failed to set up pod "pod-with-eip_default" network: CmdAdd (shim): CNI request failed with status 400: '&{ContainerID:aec9ab6585531f87f599e2174b56f8c5e0c486a8edf1be3d7bcdda6b5045a63d Netns:/proc/60758/ns/net IfName:eth0 Args:IgnoreUnknown=1;K8S_POD_NAMESPACE=default;K8S_POD_NAME=pod-with-eip;K8S_POD_INFRA_CONTAINER_ID=aec9ab6585531f87f599e2174b56f8c5e0c486a8edf1be3d7bcdda6b5045a63d Path: StdinData:[123 34 99 97 112 97 98 105 108 105 116 105 101 115 34 58 123 34 112 111 114 116 77 97 112 112 105 110 103 115 34 58 116 114 117 101 125 44 34 99 108 117 115 116 101 114 78 101 116 119 111 114 107 34 58 34 47 104 111 115 116 47 101 116 99 47 99 110 105 47 110 101 116 46 100 47 48 49 45 107 117 98 101 45 111 118 110 46 99 111 110 102 108 105 115 116 34 44 34 99 110 105 86 101 114 115 105 111 110 34 58 34 48 46 51 46 49 34 44 34 108 111 103 76 101 118 101 108 34 58 34 118 101 114 98 111 115 101 34 44 34 110 97 109 101 34 58 34 109 117 108 116 117 115 45 99 110 105 45 110 101 116 119 111 114 107 34 44 34 114 117 110 116 105 109 101 67 111 110 102 105 103 34 58 123 34 112 111 114 116 77 97 112 112 105 110 103 115 34 58 91 93 125 44 34 115 111 99 107 101 116 68 105 114 34 58 34 47 104 111 115 116 47 114 117 110 47 109 117 108 116 117 115 47 34 44 34 116 121 112 101 34 58 34 109 117 108 116 117 115 45 115 104 105 109 34 44 10 32 32 32 32 34 99 104 114 111 111 116 68 105 114 34 58 32 34 47 104 111 115 116 114 111 111 116 34 44 10 32 32 32 32 34 99 111 110 102 68 105 114 34 58 32 34 47 104 111 115 116 47 101 116 99 47 99 110 105 47 110 101 116 46 100 34 44 10 32 32 32 32 34 108 111 103 76 101 118 101 108 34 58 32 34 118 101 114 98 111 115 101 34 44 10 32 32 32 32 34 115 111 99 107 101 116 68 105 114 34 58 32 34 47 104 111 115 116 47 114 117 110 47 109 117 108 116 117 115 47 34 44 10 32 32 32 32 34 99 110 105 86 101 114 115 105 111 110 34 58 32 34 48 46 51 46 49 34 44 10 32 32 32 32 34 99 110 105 67 111 110 102 105 103 68 105 114 34 58 32 34 47 104 111 115 116 47 101 116 99 47 99 110 105 47 110 101 116 46 100 34 44 10 32 32 32 32 34 109 117 108 116 117 115 67 111 110 102 105 103 70 105 108 101 34 58 32 34 97 117 116 111 34 44 10 32 32 32 32 34 109 117 108 116 117 115 65 117 116 111 99 111 110 102 105 103 68 105 114 34 58 32 34 47 104 111 115 116 47 101 116 99 47 99 110 105 47 110 101 116 46 100 34 10 125 10]} {ContainerID:aec9ab6585531f87f599e2174b56f8c5e0c486a8edf1be3d7bcdda6b5045a63d Netns:/proc/60758/ns/net IfName:eth0 Args:IgnoreUnknown=1;K8S_POD_NAMESPACE=default;K8S_POD_NAME=pod-with-eip;K8S_POD_INFRA_CONTAINER_ID=aec9ab6585531f87f599e2174b56f8c5e0c486a8edf1be3d7bcdda6b5045a63d Path: StdinData:[123 34 99 97 112 97 98 105 108 105 116 105 101 115 34 58 123 34 112 111 114 116 77 97 112 112 105 110 103 115 34 58 116 114 117 101 125 44 34 99 108 117 115 116 101 114 78 101 116 119 111 114 107 34 58 34 47 104 111 115 116 47 101 116 99 47 99 110 105 47 110 101 116 46 100 47 48 49 45 107 117 98 101 45 111 118 110 46 99 111 110 102 108 105 115 116 34 44 34 99 110 105 86 101 114 115 105 111 110 34 58 34 48 46 51 46 49 34 44 34 108 111 103 76 101 118 101 108 34 58 34 118 101 114 98 111 115 101 34 44 34 110 97 109 101 34 58 34 109 117 108 116 117 115 45 99 110 105 45 110 101 116 119 111 114 107 34 44 34 114 117 110 116 105 109 101 67 111 110 102 105 103 34 58 123 34 112 111 114 116 77 97 112 112 105 110 103 115 34 58 91 93 125 44 34 115 111 99 107 101 116 68 105 114 34 58 34 47 104 111 115 116 47 114 117 110 47 109 117 108 116 117 115 47 34 44 34 116 121 112 101 34 58 34 109 117 108 116 117 115 45 115 104 105 109 34 44 10 32 32 32 32 34 99 104 114 111 111 116 68 105 114 34 58 32 34 47 104 111 115 116 114 111 111 116 34 44 10 32 32 32 32 34 99 111 110 102 68 105 114 34 58 32 34 47 104 111 115 116 47 101 116 99 47 99 110 105 47 110 101 116 46 100 34 44 10 32 32 32 32 34 108 111 103 76 101 118 101 108 34 58 32 34 118 101 114 98 111 115 101 34 44 10 32 32 32 32 34 115 111 99 107 101 116 68 105 114 34 58 32 34 47 104 111 115 116 47 114 117 110 47 109 117 108 116 117 115 47 34 44 10 32 32 32 32 34 99 110 105 86 101 114 115 105 111 110 34 58 32 34 48 46 51 46 49 34 44 10 32 32 32 32 34 99 110 105 67 111 110 102 105 103 68 105 114 34 58 32 34 47 104 111 115 116 47 101 116 99 47 99 110 105 47 110 101 116 46 100 34 44 10 32 32 32 32 34 109 117 108 116 117 115 67 111 110 102 105 103 70 105 108 101 34 58 32 34 97 117 116 111 34 44 10 32 32 32 32 34 109 117 108 116 117 115 65 117 116 111 99 111 110 102 105 103 68 105 114 34 58 32 34 47 104 111 115 116 47 101 116 99 47 99 110 105 47 110 101 116 46 100 34 10 125 10]} ERRORED: error configuring pod [default/pod-with-eip] networking: [default/pod-with-eip/:kube-ovn]: error adding container to network "kube-ovn": plugin type="kube-ovn" failed (add): RPC failed; request ip return 500 route is not ready for pod default/pod-with-eip provider ovn, please see kube-ovn-controller logs to find errors

kube-ovn-controller log

image

bobz965 commented 1 year ago

当 enable_eip_snat 复用已存在的 provider network vlan subnet 时，可以不用再指定网关，直接从公网子网读取网关作为下一跳ip