search

ovn-org / ovn

Open Virtual Network
Apache License 2.0
521 stars 254 forks source link

router witch always_learn_from_arp_request parameters work abnormally，ovs flow table is very very huge #135

Open zhanrox2 opened 2 years ago

zhanrox2 commented 2 years ago

Steps to reproduce:

Configure two logical routers (with distribute gateway router port) and connect them to a logical switch. Set options:always_learn_from_arp_request=false on both routers.

But OVN will still be based on ARP and IPv6 Neighbor Discovery packets , then generated entry still be put into the mac _binding table，and generate corresponding flow entries in tables 66 and tables 67 corresponding to OVS.

ovn-nbctl ls-add edge1
ovn-nbctl lsp-add edge1 localnet-port1
ovn-nbctl lsp-set-addresses localnet-port1 unknown
ovn-nbctl lsp-set-type localnet-port1 localnet
ovn-nbctl lsp-set-options localnet-port1 network_name=underlay

ovn-nbctl lsp-add edge1 edge1-nat1
ovn-nbctl lsp-set-type edge1-nat1 router
ovn-nbctl lsp-set-addresses edge1-nat1 00:00:0a:01:01:01
ovn-nbctl lsp-set-options edge1-nat1 router-port=nat1-edge1

ovn-nbctl lsp-add edge1 edge1-nat2
ovn-nbctl lsp-set-type edge1-nat2 router
ovn-nbctl lsp-set-addresses edge1-nat2 00:00:0a:01:01:02
ovn-nbctl lsp-set-options edge1-nat2 router-port=nat2-edge1

ovn-nbctl ha-chassis-group-add grp001
ovn-nbctl ha-chassis-group-add-chassis grp001 0ac970d2-a366-4cbb-a196-bb73b011bcd4 20
ovn-nbctl ha-chassis-group-add-chassis grp001 27fa1df9-d5a0-4da1-bc2d-9111b2f14639 30

ovn-nbctl lr-add r1
ovn-nbctl lrp-add r1 r1-nat1 00:00:ac:10:01:01 172.16.1.1/24 peer=nat1-r1
ovn-nbctl lrp-add r1 r1-sw1 00:00:c0:a8:01:fe 192.168.1.254/24

ovn-nbctl ls-add sw1
ovn-nbctl lsp-add sw1 port1
ovn-nbctl lsp-set-addresses port1 "00:00:c0:a8:01:01 192.168.1.1"

ovn-nbctl lsp-add sw1 sw1-r1
ovn-nbctl lsp-set-type sw1-r1 router
ovn-nbctl lsp-set-addresses sw1-r1 00:00:c0:a8:01:fe
ovn-nbctl lsp-set-options sw1-r1 router-port=r1-sw1

ovn-nbctl lr-add nat1
ovn-nbctl lrp-add nat1 nat1-r1 00:00:ac:10:01:02 172.16.1.2/24 peer=r1-nat1
ovn-nbctl lrp-add nat1 nat1-edge1 00:00:0a:01:01:01 10.1.1.1/24
ovn-nbctl set Logical_Router_Port  nat1-edge1 ha_chassis_group=5cb3a984-7f98-4084-a73c-bdc18d129a94

ovn-nbctl lr-nat-add nat1 snat 10.1.1.1 192.168.0.0/16
ovn-nbctl lr-route-add nat1 "0.0.0.0/0" 10.1.1.254
ovn-nbctl lr-route-add nat1 "192.168.0.0/16" 172.16.1.1
ovn-nbctl lr-route-add r1  "0.0.0.0/0"  172.16.1.2

ovn-nbctl lr-add r2
ovn-nbctl lrp-add r2 r2-nat2 00:00:ac:10:01:01 172.16.1.1/24 peer=nat2-r2
ovn-nbctl lrp-add r2 r2-sw2 00:00:c0:a8:01:fe 192.168.1.254/24

ovn-nbctl ls-add sw2
ovn-nbctl lsp-add sw2 port2
ovn-nbctl lsp-set-addresses port2 "00:00:c0:a8:01:01 192.168.1.1"

ovn-nbctl lsp-add sw2 sw2-r2
ovn-nbctl lsp-set-type sw2-r2 router
ovn-nbctl lsp-set-addresses sw2-r2 00:00:c0:a8:01:fe
ovn-nbctl lsp-set-options sw2-r2 router-port=r2-sw2

ovn-nbctl lr-add nat2
ovn-nbctl lrp-add nat2 nat2-r2 00:00:ac:10:01:02 172.16.1.2/24 peer=r2-nat2
ovn-nbctl lrp-add nat2 nat2-edge1 00:00:0a:01:01:02 10.1.1.2/24
ovn-nbctl set Logical_Router_Port  nat2-edge1 ha_chassis_group=5cb3a984-7f98-4084-a73c-bdc18d129a94

ovn-nbctl lr-nat-add nat2 snat 10.1.1.2 192.168.0.0/16
ovn-nbctl lr-route-add nat2 "0.0.0.0/0" 10.1.1.254
ovn-nbctl lr-route-add nat2 "192.168.0.0/16" 172.16.1.1
ovn-nbctl lr-route-add r2  "0.0.0.0/0"  172.16.1.2

ovn-nbctl set logical_router r1 options:always_learn_from_arp_request=false
ovn-nbctl set logical_router r2 options:always_learn_from_arp_request=false
ovn-nbctl set logical_router nat1 options:always_learn_from_arp_request=false
ovn-nbctl set logical_router nat2 options:always_learn_from_arp_request=false

ovn-nbctl set logical_router r1 options:dynamic_neigh_routers=true
ovn-nbctl set logical_router r2 options:dynamic_neigh_routers=true
ovn-nbctl set logical_router nat1 options:dynamic_neigh_routers=true
ovn-nbctl set logical_router nat2 options:dynamic_neigh_routers=true

ip netns add c1
ip link add veth0 type veth peer name veth1
ip link set veth1 netns c1
ip netns exec c1 ip link set lo up
ip netns exec c1 ip link set veth1 up
ip netns exec c1 ip link set veth1 address 00:00:c0:a8:01:01
ip netns exec c1 ip addr add 192.168.1.1/24 dev veth1
ip netns exec c1 ip route add default via 192.168.1.254
ip link set veth0 up
ovs-vsctl add-port br-int veth0
ovs-vsctl set Interface veth0 external_ids:iface-id=port1

ip netns add c2
ip link add veth2 type veth peer name veth3
ip link set veth3 netns c2
ip netns exec c2 ip link set lo up
ip netns exec c2 ip link set veth3 up
ip netns exec c2 ip link set veth3 address 00:00:c0:a8:01:01
ip netns exec c2 ip addr add 192.168.1.1/24 dev veth3
ip netns exec c2 ip route add default via 192.168.1.254
ip link set veth2 up
ovs-vsctl add-port br-int veth2
ovs-vsctl set Interface veth2 external_ids:iface-id=port2

You will find the corresponding records in the mac_binding table as shown below

# ovn-sbctl list mac_binding
_uuid               : 8980d0e8-58f3-470d-b5b5-cafc90df2a21
datapath            : 3284257b-b4aa-4e1c-8aa6-11a7d9b4e540
ip                  : "::"
logical_port        : nat1-edge1
mac                 : "00:00:00:00:00:00"

_uuid               : 677116e0-8cc4-402b-9dc7-45836ec0406e
datapath            : 8ab898d2-2504-4e61-8cd7-df353334818d
ip                  : "::"
logical_port        : r1-sw1
mac                 : "00:00:00:00:00:00"

_uuid               : 5a669562-8b54-4232-8b6f-5c4e2e023113
datapath            : 3284257b-b4aa-4e1c-8aa6-11a7d9b4e540
ip                  : "10.1.1.254"
logical_port        : nat1-edge1
mac                 : "00:0c:29:dd:94:53"

_uuid               : e19cbe72-f243-4ba0-9115-299928654a8f
datapath            : 17a414dd-2691-46a1-baf2-5a560fc2f725
ip                  : "::"
logical_port        : r2-sw2
mac                 : "00:00:00:00:00:00"

_uuid               : 30bb440a-3501-4cb4-9fe4-48cba17640d6
datapath            : 3284257b-b4aa-4e1c-8aa6-11a7d9b4e540
ip                  : "10.1.1.2"
logical_port        : nat1-edge1
mac                 : "00:00:0a:01:01:02"

_uuid               : ae613e54-42b7-461b-9f51-fcae69f239e9
datapath            : 17a414dd-2691-46a1-baf2-5a560fc2f725
ip                  : "192.168.1.1"
logical_port        : r2-sw2
mac                 : "00:00:c0:a8:01:01"

_uuid               : f480e2d5-a0ef-4484-83f6-80f4ba436cfb
datapath            : 38be8a59-09b6-4ef9-9af2-f1c8b472e611
ip                  : "10.1.1.254"
logical_port        : nat2-edge1
mac                 : "00:0c:29:dd:94:53"

_uuid               : 8f063a7d-bcc2-45a6-bf99-41bddab2130e
datapath            : 38be8a59-09b6-4ef9-9af2-f1c8b472e611
ip                  : "::"
logical_port        : nat2-edge1
mac                 : "00:00:00:00:00:00"

_uuid               : 7c455b46-99d6-4291-a1c2-edf0ef30fc52
datapath            : 8ab898d2-2504-4e61-8cd7-df353334818d
ip                  : "192.168.1.1"
logical_port        : r1-sw1
mac                 : "00:00:c0:a8:01:01"

And even if the table is cleared with the command "ovn-sbctl --all destroy mac_binding", the corresponding entry will be automatically generated after communication.

I think if this option is set to false, the LRPs connecting switches would not learn mac-bindings for overlay VIFs if the pod sends GARP requests, which would add unnecessary entries in SB and increase control plane churns unnecessarily.

In addition, I also found that if there are N distributed gateway ports connected to the same edge switch, the number of OVS flow tables (table66 and tables 67) is close to the N*N Cartesian product, which is very large.

Can someone tell me how I can handle this next ？

zhanrox2 commented 2 years ago

OVN 22.03 , OVS 2.17.0

almusil commented 2 years ago

Hi, those that have empty ip or empty mac should be handled by commits 0a4bc20 and 80187a8.

Second issue seems to be the order of operation, you are setting options:always_learn_from_arp_request=false after ovn-nbctl lr-nat-add nat1 snat 10.1.1.1 192.168.0.0/16 and ovn-nbctl lr-nat-add nat2 snat 10.1.1.2 192.168.0.0/16, which creates MAC binding row unless always_learn_from_arp_request=false [0]. I am not sure where the 10.1.1.254 is created, but there is chance you are getting GARP from the localport before the option is set.

For the second point there is ongoing work on aging mechanism for MAC binding, that should prevent the MAC binding table to grow infinitely.

[0] https://github.com/ovn-org/ovn/blob/main/controller/pinctrl.c#L4415