oilbeater
commented
1 year ago I cannot reproduce this issue. Did you change other config about subnet or kube-ovn-controller
Closed bobz965 closed 1 year ago
oilbeater
commented
1 year ago I cannot reproduce this issue. Did you change other config about subnet or kube-ovn-controller
bobz965
commented
1 year ago I cannot reproduce this issue. Did you change other config about subnet or kube-ovn-controller
i rebuild my env today with latest 1.11
# k get deployment -A -o wide | grep kube-ovn-controller
kube-system kube-ovn-controller 1/1 1 1 178m kube-ovn-controller kubeovn/kube-ovn:v1.11.0 app=kube-ovn-controller
[root@pc-node-1 ~]# k get daemonset -A -o wide | grep ovn
kube-system kube-ovn-cni 3 3 3 3 3 kubernetes.io/os=linux 179m cni-server kubeovn/kube-ovn:v1.11.0 app=kube-ovn-cni
kube-system kube-ovn-pinger 3 3 3 3 3 kubernetes.io/os=linux 179m pinger kubeovn/kube-ovn:v1.11.0 app=kube-ovn-pinger
kube-system ovs-ovn 3 3 3 3 3 kubernetes.io/os=linux 179m openvswitch kubeovn/kube-ovn:v1.11.0 app=ovs
# 1. enable ovn-cluster eip_snat
[root@pc-node-1 01-test-old-enable-eip-snat]# cat 00-centralized-external-gw-no-ip.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: ovn-external-gw-config
namespace: kube-system
data:
enable-external-gw: "true"
external-gw-nodes: "pc-node-1,pc-node-2,pc-node-3" # 目前先测试一个节点
type: "centralized" # 默认为 centralized 如果使用 distributed，则集群所有节点都需要有同名网卡来承担网关功能。
external-gw-nic: "vlan"
external-gw-addr: "10.5.204.254/24" # 物理网关的ip
# 2. test ecmp
# cat default-ecmp.yaml
apiVersion: kubeovn.io/v1
kind: Subnet
metadata:
name: default-ecmp-subnet
spec:
protocol: IPv4
cidrBlock: 192.168.0.0/24
default: false
gatewayNode: "pc-node-1,pc-node-2,pc-node-3"
gatewayType: centralized
natOutgoing: true
vpc: ovn-cluster
other operations:
[root@pc-node-1 00-provider-net-work]# cat 01-provider-network.yaml
apiVersion: kubeovn.io/v1
kind: ProviderNetwork
metadata:
name: external204
spec:
defaultInterface: vlan
[root@pc-node-1 00-provider-net-work]# cat 02-vlan.yaml
apiVersion: kubeovn.io/v1
kind: Vlan
metadata:
name: vlan204
spec:
id: 204
provider: external204
[root@pc-node-1 00-provider-net-work]# cat 03-vlan-subnet.yaml
apiVersion: kubeovn.io/v1
kind: Subnet
metadata:
name: external204
spec:
protocol: IPv4
cidrBlock: 10.5.204.0/24
gateway: 10.5.204.254
vlan: vlan204
excludeIps:
- 10.5.204.1..10.5.204.100
[root@pc-node-1 master]# diff install.sh /tmp/install.sh
13,14c13,14
< ENABLE_LB=${ENABLE_LB:-false}
< ENABLE_NP=${ENABLE_NP:-false}
---
> ENABLE_LB=${ENABLE_LB:-true}
> ENABLE_NP=${ENABLE_NP:-true}
26c26
< IFACE=${IFACE:-tunnel}
---
> IFACE=${IFACE:-}
74c74
< VLAN_ID="0"
---
> VLAN_ID="100"
2998,3000d2997
< - --external-gateway-vlanid=204
< - --external-gateway-switch=external204
< - --enable-ecmp=true
3124,3125d3120
< - --external-gateway-switch=external204
<
# connect ovn-cluster with provider networks
# kubectl ko nbctl show
switch 0988dc35-1557-49a5-a86d-33e669995fd1 (ovn-default)
port kube-ovn-pinger-5wtzl.kube-system
addresses: ["00:00:00:26:F5:87 10.16.0.18"]
port pod-with-fip.default
addresses: ["00:00:00:DD:77:C5 10.16.0.2"]
port coredns-7f6f7d5658-tvjd5.kube-system
addresses: ["00:00:00:EB:5C:5E 10.16.0.11"]
port kube-ovn-pinger-qzpph.kube-system
addresses: ["00:00:00:E9:4D:8C 10.16.0.17"]
port kube-ovn-pinger-lhpbr.kube-system
addresses: ["00:00:00:A7:E7:89 10.16.0.14"]
port ovn-default-ovn-cluster
type: router
router-port: ovn-cluster-ovn-default
port coredns-7f6f7d5658-mth5z.kube-system
addresses: ["00:00:00:4E:ED:C9 10.16.0.12"]
port dns-autoscaler-546cbcc6fc-n59r2.kube-system
addresses: ["00:00:00:C9:A4:2D 10.16.0.13"]
port kube-ovn-pinger-2t52w.kube-system
addresses: ["00:00:00:50:43:75 10.16.0.15"]
port kube-ovn-pinger-d5d2j.kube-system
addresses: ["00:00:00:84:23:75 10.16.0.16"]
switch ca2dcd59-cae7-45b3-ac38-d21aca94c254 (external)
port localnet.external
type: localnet
addresses: ["unknown"]
port external-ovn-cluster
type: router
router-port: ovn-cluster-external
switch 8ffbf1e2-bd46-436c-8303-e40b7adc9ab0 (join)
port node-hci-dev-mst-3
addresses: ["00:00:00:C2:37:AC 100.64.0.2"]
port node-hci-dev-mst-2
addresses: ["00:00:00:62:81:ED 100.64.0.3"]
port node-hci-dev-work-2
addresses: ["00:00:00:8A:92:24 100.64.0.5"]
port node-hci-dev-mst-1
addresses: ["00:00:00:97:10:0D 100.64.0.6"]
port join-ovn-cluster
type: router
router-port: ovn-cluster-join
port node-hci-dev-work-1
addresses: ["00:00:00:7D:6D:95 100.64.0.4"]
switch 4950cfe6-e549-4eed-a717-7c891495df93 (default-ecmp-subnet)
port default-ecmp-subnet-ovn-cluster
type: router
router-port: ovn-cluster-default-ecmp-subnet
port ecmp-busybox.default
addresses: ["00:00:00:32:7B:48 192.168.0.3"]
router f6af44a2-6303-4581-81d8-efb5120c6e4a (ovn-cluster)
port ovn-cluster-external
mac: "fa:16:3e:08:82:34"
networks: ["172.20.10.2/24"]
gateway chassis: [7bddd50c-bdc5-4252-9fdc-dd21e1da0c75 05877115-cade-4b31-97e8-f69eec16fece f70fad61-1069-4bd7-bf3a-7253250028c6 5b3f7f12-2ec6-46bc-8e98-44b29c0a6c1d 61c9dc8f-57e5-4bc4-b745-4b992137471c]
port ovn-cluster-default-ecmp-subnet
mac: "00:00:00:FE:1A:EB"
networks: ["192.168.0.1/24"]
port ovn-cluster-join
mac: "00:00:00:C9:4B:D0"
networks: ["100.64.0.1/16"]
port ovn-cluster-ovn-default
mac: "00:00:00:90:5A:D8"
networks: ["10.16.0.1/16"]
nat 91c23c42-1157-4a97-930f-089179292942
external ip: "172.20.10.50"
logical ip: "10.16.0.2"
type: "dnat_and_snat"
hongzhen-ma
commented
1 year ago 
策略路由的文档中提到，唯一确定一条策略路由，是通过 priority and match，对于ECMP策略路由，match的表达式是 match := fmt.Sprintf("%s.src == %s", ipSuffix, cidr)，因此下一跳变更，是无法区分出来这个策略路由需要更新的。 所以对于ECMP路由，更新的方式，是删除原来的策略路由，再重建新的策略路由。 这个可能是对kube-ovn pod 重建，这种情况有影响。
bobz965
commented
1 year ago 策略路由的文档中提到，唯一确定一条策略路由，是通过 priority and match，对于ECMP策略路由，match的表达式是 match := fmt.Sprintf("%s.src == %s", ipSuffix, cidr)，因此下一跳变更，是无法区分出来这个策略路由需要更新的。 所以对于ECMP路由，更新的方式，是删除原来的策略路由，再重建新的策略路由。 这个可能是对，除ovn-central之外的kube-ovn pod 重建，这种情况有影响。
感谢， 我继续看下，我增加了一些log，确实有看到一些在add 流程中触发check并删除路由的情况
bobz965
commented
1 year ago 
目前问题查的差不多了，只剩一个启动时好像会重置路由的操作。 我先提个draft，下周再继续看下把这个步骤看能不能也避免下
Expected Behavior
keep ecmp lr policy route id static
Actual Behavior
Steps to Reproduce the Problem
master