search

facebookincubator / katran

A high performance layer 4 load balancer
GNU General Public License v2.0
4.75k stars 504 forks source link

katran stops working after running ip rule add #169

Closed yaa110 closed 2 years ago

yaa110 commented 2 years ago

Katran (maybe ipip tunnel) stops working after running the following command:

echo "10 d10" >> /etc/iproute2/rt_tables
ip rule a from 10.1.1.2 table 10 # breaking command

It is not working again after deleting table and rule (A reboot is needed).

uname -r
4.15.0-142-generic
ip tun
gre0: gre/ip remote any local any ttl inherit nopmtudisc
ipip0: any/ip remote any local any ttl inherit
ipip0: any/ip remote any local any ttl inherit
ip r | grep default
default via 79.x.x.x dev bond0 proto static

ip n | grep 79.x.x.x
79.x.x.x dev bond0 lladdr 18:8b:9d:3d:62:3f REACHABLE
ip rule
0:  from all lookup local 
32766:  from all lookup main 
32767:  from all lookup default
ip a
5: ens3f0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1450 xdp/id:18 qdisc mq master bond0 state UP group default qlen 1000
    link/ether 1e:93:33:af:d4:b4 brd ff:ff:ff:ff:ff:ff
7: ens3f1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1450 xdp/id:16 qdisc mq master bond0 state UP group default qlen 1000
    link/ether 1e:93:33:af:d4:b4 brd ff:ff:ff:ff:ff:ff
8: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default qlen 1000
    link/ether 1e:93:33:af:d4:b4 brd ff:ff:ff:ff:ff:ff
    inet 79.x.x.x/28 brd 79.x.x.x scope global bond0
       valid_lft forever preferred_lft forever
    inet 172.x.x.x/29 brd 172.x.x.x scope global bond0
       valid_lft forever preferred_lft forever
    inet 172.x.x.x/29 brd 172.x.x.x scope global bond0
       valid_lft forever preferred_lft forever
    inet6 fe80::1c93:33ff:feaf:d4b4/64 scope link 
       valid_lft forever preferred_lft forever
9: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
10: ipip0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
11: ip6tnl0@NONE: <NOARP> mtu 1452 qdisc noop state DOWN group default qlen 1000
    link/tunnel6 :: brd ::
12: ipip60@NONE: <NOARP,UP,LOWER_UP> mtu 1452 qdisc noqueue state UNKNOWN group default qlen 1000
    link/tunnel6 :: brd ::
    inet6 fe80::e466:e1ff:fe41:26c4/64 scope link 
       valid_lft forever preferred_lft forever
13: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    link/ether 02:42:72:7c:29:e1 brd ff:ff:ff:ff:ff:ff
    inet 172.x.x.x/16 brd 172.x.x.x scope global docker0
       valid_lft forever preferred_lft forever

Katran is run using docker:

version: '3.0'

services:
  katran_ens3f1:
    image: katran:latest
    container_name: katran_ens3f1
    restart: always
    privileged: true
    cap_add:
      - NET_ADMIN
      - SYS_ADMIN
    network_mode: "host"
    volumes:
      - "/sys/fs/bpf:/sys/fs/bpf"
    command: ["-default_mac", "18:8b:9d:3d:62:3f", "-intf", "ens3f1", "-server", "127.0.0.1:50051", "-map_path", "/sys/fs/bpf/ens3f1/root_array", "-prog_pos", "2"]
  katran_ens3f0:
    image: katran:latest
    container_name: katran_ens3f0
    restart: always
    privileged: true
    cap_add:
      - NET_ADMIN
      - SYS_ADMIN
    network_mode: "host"
    volumes:
      - "/sys/fs/bpf:/sys/fs/bpf"
    command: ["-default_mac", "18:8b:9d:3d:62:3f", "-intf", "ens3f0", "-server", "127.0.0.1:50052", "-map_path", "/sys/fs/bpf/ens3f0/root_array", "-prog_pos", "2"]
iptables -S
-P INPUT DROP
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-N DOCKER-USER
-N YYY-BGP
-N YYY-IPIP
-N YYY-SPOOF
-A INPUT -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A INPUT -p icmp -j ACCEPT
-A INPUT -p gre -j ACCEPT
-A INPUT -s 10.0.0.0/8 -p tcp -j ACCEPT
-A INPUT -s 10.0.0.0/8 -p udp -j ACCEPT
-A INPUT -d 10.0.0.0/8 -p tcp -j ACCEPT
-A INPUT -d 10.0.0.0/8 -p udp -j ACCEPT
-A INPUT -p ipencap -j YYY-IPIP
-A INPUT -p icmp -m comment --comment app-icmp
-A INPUT -i lo -j ACCEPT
-A INPUT -p icmp -j YYY-BGP
-A INPUT -s 0.0.0.0/8 -j YYY-SPOOF
-A INPUT -s 10.0.0.0/8 -j YYY-SPOOF
-A INPUT -s 100.64.0.0/10 -j YYY-SPOOF
-A INPUT -s 127.0.0.0/8 -j YYY-SPOOF
-A INPUT -s 169.254.0.0/16 -j YYY-SPOOF
-A INPUT -s 172.16.0.0/12 -j YYY-SPOOF
-A INPUT -m conntrack --ctstate INVALID -m comment --comment drop-invalid -j DROP
-A INPUT -p icmp -m icmp --icmp-type 8 -m length ! --length 1:1500 -m comment --comment drop-icmp-huge -j DROP
-A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A INPUT -m comment --comment drop-default
-A FORWARD -j DOCKER-USER
-A OUTPUT -p icmp -m comment --comment app-icmp
-A DOCKER-USER -j RETURN
-A YYY-SPOOF -m comment --comment drop-spoofed -j DROP
iptables -t nat -S
-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
iptables -t mangle -S
-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
iptables -t raw -S
-P PREROUTING ACCEPT
-P OUTPUT ACCEPT
sysctl -a | grep rp_filter
net.ipv4.conf.all.arp_filter = 0
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.bond0.arp_filter = 0
net.ipv4.conf.bond0.rp_filter = 0
net.ipv4.conf.default.arp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.docker0.arp_filter = 0
net.ipv4.conf.docker0.rp_filter = 0
net.ipv4.conf.eno1.arp_filter = 0
net.ipv4.conf.eno1.rp_filter = 0
net.ipv4.conf.eno2.arp_filter = 0
net.ipv4.conf.eno2.rp_filter = 0
net.ipv4.conf.eno3.arp_filter = 0
net.ipv4.conf.eno3.rp_filter = 0
net.ipv4.conf.eno4.arp_filter = 0
net.ipv4.conf.eno4.rp_filter = 0
net.ipv4.conf.ens3f0.arp_filter = 0
net.ipv4.conf.ens3f0.rp_filter = 0
net.ipv4.conf.ens3f1.arp_filter = 0
net.ipv4.conf.ens3f1.rp_filter = 0
net.ipv4.conf.ip6tnl0.arp_filter = 0
net.ipv4.conf.ip6tnl0.rp_filter = 0
net.ipv4.conf.ipip0.arp_filter = 0
net.ipv4.conf.ipip0.rp_filter = 0
net.ipv4.conf.ipip60.arp_filter = 0
net.ipv4.conf.ipip60.rp_filter = 0
net.ipv4.conf.lo.arp_filter = 0
net.ipv4.conf.lo.rp_filter = 0
net.ipv4.conf.tunl0.arp_filter = 0
net.ipv4.conf.tunl0.rp_filter = 0