Open laik opened 2 years ago
debian01 192.168.122.100 (控制节点) debian02 192.168.122.101 (计算节点) debian03 192.168.122.102 ditto
版本: Distributor ID: Debian Description: Debian GNU/Linux 12 (bookworm) Release: 12 Codename: bookworm
本次实验目标实现基于k8s中使用ovn组成网络实验,参考kube-ovn
[ovn-cluster(R)]
[join 100.64.0.0/24(S)] [ovn-default 192.168.1.0/24(S)] [ovn-subnet-2 192.168.2.0/24(S)]
sudo apt install net-tools
sudo apt-get install python-six openssl -y
sudo apt-get install openvswitch-switch openvswitch-common -y
sudo systemctl disable apparmor
sudo apt-get install ovn-central ovn-common ovn-host -y
sudo apt-get install ovn-host ovn-common -y
ovn-nbctl set-connection ptcp:6641:192.168.122.100
ovn-sbctl set-connection ptcp:6642:192.168.122.100
// debian01上ovn-controller连接南向数据库
// ovn-remote:指定南向数据库连接地址
// ovn-encap-ip:指定ovs/controller本地ip
// ovn-encap-type:指定隧道协议,这里用的是geneve
// system-id:节点标识
// 控制节点数据库主机 192.168.122.100
ovs-vsctl set Open_vSwitch . external-ids:ovn-remote="tcp:192.168.122.100:6642" external-ids:ovn-encap-ip="$(hostname -I | awk '{print $1}')" external-ids:ovn-encap-type=geneve external-ids:system-id=$(hostname)
root@debian01:~# netstat -anlpt | grep 6642
tcp 0 0 192.168.122.100:6642 0.0.0.0:* LISTEN 3014/ovsdb-server
tcp 0 0 192.168.122.100:6642 192.168.122.100:60970 ESTABLISHED 3014/ovsdb-server
tcp 0 0 192.168.122.100:6642 192.168.122.102:60364 ESTABLISHED 3014/ovsdb-server
tcp 0 0 192.168.122.100:6642 192.168.122.101:51894 ESTABLISHED 3014/ovsdb-server
tcp 0 0 192.168.122.100:60970 192.168.122.100:6642 ESTABLISHED 3102/ovn-controller
root@debian01:~# ovs-vsctl --columns external_ids list open_vswitch
external_ids : {hostname=debian01, ovn-encap-ip="192.168.122.100", ovn-encap-type=geneve, ovn-remote="tcp:192.168.122.100:6642", rundir="/var/run/openvswitch", system-id=debian01}
root@debian02:~# ovs-vsctl --columns external_ids list open_vswitch
external_ids : {hostname=debian02, ovn-encap-ip="192.168.122.101", ovn-encap-type=geneve, ovn-remote="tcp:192.168.122.100:6642", rundir="/var/run/openvswitch", system-id=debian02}
root@debian03:~# ovs-vsctl --columns external_ids list open_vswitch
external_ids : {hostname=debian03, ovn-encap-ip="192.168.122.102", ovn-encap-type=geneve, ovn-remote="tcp:192.168.122.100:6642", rundir="/var/run/openvswitch", system-id=debian03}
// debian01上查看南向数据库
root@debian01:~# ovn-sbctl show
Chassis debian03
hostname: debian03
Encap geneve
ip: "192.168.122.102"
options: {csum="true"}
Chassis debian01
hostname: debian01
Encap geneve
ip: "192.168.122.100"
options: {csum="true"}
Chassis debian02
hostname: debian02
Encap geneve
ip: "192.168.122.101"
options: {csum="true"}
//添加 ns 在所有节点,创建一个名为ns1的namespace,并将veth11端口添加到br-int,将veth12端口添加到ns1
// debian01 上执行
ip netns add ns1
ip link add veth11 type veth peer name veth12
ip link set veth12 netns ns1
ip link set veth11 up
ip netns exec ns1 ip link set veth12 address 00:00:00:00:00:01
ip netns exec ns1 ip link set veth12 up
ip netns exec ns1 ip link set lo up
ovs-vsctl add-port br-int veth11
ip netns exec ns1 ip addr add 192.168.1.10/24 dev veth12
// debian02上执行
ip netns add ns1
ip link add veth11 type veth peer name veth12
ip link set veth12 netns ns1
ip link set veth11 up
ip netns exec ns1 ip link set veth12 address 00:00:00:00:00:02
ip netns exec ns1 ip link set veth12 up
ip netns exec ns1 ip link set lo up
ovs-vsctl add-port br-int veth11
ip netns exec ns1 ip addr add 192.168.1.20/24 dev veth12
// debian03上执行
ip netns add ns1
ip link add veth11 type veth peer name veth12
ip link set veth12 netns ns1
ip link set veth11 up
ip netns exec ns1 ip link set veth12 address 00:00:00:00:00:03
ip netns exec ns1 ip link set veth12 up
ip netns exec ns1 ip link set lo up
ovs-vsctl add-port br-int veth11
ip netns exec ns1 ip addr add 192.168.1.30/24 dev veth12
// 查看每个节点上br-int交换机信息
root@debian01:~# ovs-vsctl show
061a31bd-f8da-47ac-8044-6c2d36b1d622
Bridge br-int
fail_mode: secure
datapath_type: system
Port br-int
Interface br-int
type: internal
Port ovn-debian-1
Interface ovn-debian-1
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.122.102"}
Port ovn-debian-0
Interface ovn-debian-0
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.122.101"}
Port veth11
Interface veth11
ovs_version: "3.1.0"
root@debian02:~# ovs-vsctl show
24a78807-44bc-4e30-8109-f61f1dfe017e
Bridge br-int
fail_mode: secure
datapath_type: system
Port br-int
Interface br-int
type: internal
Port ovn-debian-1
Interface ovn-debian-1
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.122.102"}
Port veth11
Interface veth11
Port ovn-debian-0
Interface ovn-debian-0
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.122.100"}
ovs_version: "3.1.0"
root@debian03:~# ovs-vsctl show
14916c30-b4a5-436b-b291-0e0ca285b399
Bridge br-int
fail_mode: secure
datapath_type: system
Port ovn-debian-0
Interface ovn-debian-0
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.122.100"}
Port ovn-debian-1
Interface ovn-debian-1
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.122.101"}
Port br-int
Interface br-int
type: internal
Port veth11
Interface veth11
ovs_version: "3.1.0"
当前,每个节点上的ns1还是不能通讯的。需要在OVN中添加逻辑交换机,并将物理网络中的br-int桥接到逻辑交换机上。
// 在debian01上执行
root@debian01:~# ip netns exec ns1 ping -c 1 192.168.1.20
PING 192.168.1.20 (192.168.1.20) 56(84) bytes of data.
From 192.168.1.10 icmp_seq=1 Destination Host Unreachable
--- 192.168.1.20 ping statistics ---
1 packets transmitted, 0 received, +1 errors, 100% packet loss, time 0ms
// 以下都在在debian01上执行
ovn-nbctl ls-add ovn-default
// 添加逻辑交换机端口,并配置与 ns 对应的veth-pair端口mac地址
ovn-nbctl lsp-add ovn-default ovn-default-debian01-ns1
ovn-nbctl lsp-set-addresses ovn-default-debian01-ns1 "00:00:00:00:00:01 192.168.1.10"
ovn-nbctl lsp-set-port-security ovn-default-debian01-ns1 00:00:00:00:00:01
ovn-nbctl lsp-add ovn-default ovn-default-debian02-ns1
ovn-nbctl lsp-set-addresses ovn-default-debian02-ns1 "00:00:00:00:00:02 192.168.1.20"
ovn-nbctl lsp-set-port-security ovn-default-debian02-ns1 00:00:00:00:00:02
ovn-nbctl lsp-add ovn-default ovn-default-debian03-ns1
ovn-nbctl lsp-set-addresses ovn-default-debian03-ns1 "00:00:00:00:00:03 192.168.1.30"
ovn-nbctl lsp-set-port-security ovn-default-debian03-ns1 00:00:00:00:00:03
// 查看逻辑交换机信息
root@debian01:~# ovn-nbctl show
switch d7ab026e-09ed-4ac7-9491-2510caa1c57d (ovn-default)
port ovn-default-debian01-ns1
addresses: ["00:00:00:00:00:01 192.168.1.10"]
port ovn-default-debian03-ns1
addresses: ["00:00:00:00:00:03 192.168.1.30"]
port ovn-default-debian02-ns1
addresses: ["00:00:00:00:00:02 192.168.1.20"]
// debian01上执行,veth11端口连接逻辑交换机端口
ovs-vsctl set interface veth11 external-ids:iface-id=ovn-default-debian01-ns1
// debian02上执行,veth11端口连接逻辑交换机端口
ovs-vsctl set interface veth11 external-ids:iface-id=ovn-default-debian02-ns1
// debian03上执行,veth11端口连接逻辑交换机端口
ovs-vsctl set interface veth11 external-ids:iface-id=ovn-default-debian03-ns1
// debian01上验证网络连通性
root@debian01:~# ip netns exec ns1 ping -c 1 192.168.1.20
PING 192.168.1.20 (192.168.1.20) 56(84) bytes of data.
64 bytes from 192.168.1.20: icmp_seq=1 ttl=64 time=2.90 ms
--- 192.168.1.20 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.898/2.898/2.898/0.000 ms
// 查看南北向数据库
root@debian01:~# ovn-nbctl show
switch d7ab026e-09ed-4ac7-9491-2510caa1c57d (ovn-default)
port ovn-default-debian01-ns1
addresses: ["00:00:00:00:00:01 192.168.1.10"]
port ovn-default-debian03-ns1
addresses: ["00:00:00:00:00:03 192.168.1.30"]
port ovn-default-debian02-ns1
addresses: ["00:00:00:00:00:02 192.168.1.20"]
root@debian01:~# ovn-sbctl show
Chassis debian03
hostname: debian03
Encap geneve
ip: "192.168.122.102"
options: {csum="true"}
Port_Binding ovn-default-debian03-ns1
Chassis debian01
hostname: debian01
Encap geneve
ip: "192.168.122.100"
options: {csum="true"}
Port_Binding ovn-default-debian01-ns1
Chassis debian02
hostname: debian02
Encap geneve
ip: "192.168.122.101"
options: {csum="true"}
Port_Binding ovn-default-debian02-ns1
// debian02上执行
ip netns add ns2
ip link add veth21 type veth peer name veth22
ip link set veth22 netns ns2
ip link set veth21 up
ip netns exec ns2 ip link set veth22 address 00:00:00:00:00:04
ip netns exec ns2 ip link set veth22 up
ip netns exec ns2 ip link set lo up
ovs-vsctl add-port br-int veth21
ip netns exec ns2 ip addr add 192.168.2.30/24 dev veth22
// 在debian01上用命令新增一个逻辑交换机,并配置好端口
ovn-nbctl ls-add ovn-subnet-2
ovn-nbctl lsp-add ovn-subnet-2 ovn-subnet-2-debian02-ns2
ovn-nbctl lsp-set-addresses ovn-subnet-2-debian02-ns2 "00:00:00:00:00:04 192.168.2.30"
ovn-nbctl lsp-set-port-security ovn-subnet-2-debian02-ns2 00:00:00:00:00:04
// debian02上执行将ovs交换机端口和ovn逻辑交换机端口匹配起来
ovs-vsctl set interface veth21 external-ids:iface-id=ovn-subnet-2-debian02-ns2
// 在debian01上用命令查看北向数据库和南向数据库信息
root@debian01:~# ovn-nbctl show
switch eff58e2c-02b8-4c24-a0f0-72e785b98781 (ovn-subnet-2)
port ovn-subnet-2-debian02-ns2
addresses: ["00:00:00:00:00:04 192.168.2.30"]
switch d7ab026e-09ed-4ac7-9491-2510caa1c57d (ovn-default)
port ovn-default-debian01-ns1
addresses: ["00:00:00:00:00:01 192.168.1.10"]
port ovn-default-debian03-ns1
addresses: ["00:00:00:00:00:03 192.168.1.30"]
port ovn-default-debian02-ns1
addresses: ["00:00:00:00:00:02 192.168.1.20"]
// 在debian01上用命令添加逻辑路由器
ovn-nbctl lr-add ovn-cluster
以下两个操作类似在传统的路由器例如cisco路由器的端口上配置ip地址
// 在debian01上用命令逻辑路由器添加连接交换机ovn-cluster-ovn-default的端口
ovn-nbctl lrp-add ovn-cluster ovn-cluster-ovn-default 00:00:00:00:11:00 192.168.1.1/24
// 在debian01上用命令逻辑路由器添加连接交换机ovn-cluster-ovn-subnet-2的端口
ovn-nbctl lrp-add ovn-cluster ovn-cluster-ovn-subnet-2 00:00:00:00:12:00 192.168.2.1/24
以上两个操作类似在传统的路由器与交换机之间连接网线
// 在debian01上用命令逻辑路由器ovn-cluster连接逻辑交换机ovn-default
ovn-nbctl lsp-add ovn-default ovn-default-ovn-cluster
ovn-nbctl lsp-set-type ovn-default-ovn-cluster router
ovn-nbctl lsp-set-addresses ovn-default-ovn-cluster "00:00:00:00:11:00 192.168.1.1"
ovn-nbctl lsp-set-options ovn-default-ovn-cluster router-port=ovn-cluster-ovn-default
// 在debian01上用命令逻辑路由器ovn-cluster连接逻辑交换机ovn-subnet-2
ovn-nbctl lsp-add ovn-subnet-2 ovn-subnet-2-ovn-cluster
ovn-nbctl lsp-set-type ovn-subnet-2-ovn-cluster router
ovn-nbctl lsp-set-addresses ovn-subnet-2-ovn-cluster "00:00:00:00:12:00 192.168.2.1"
ovn-nbctl lsp-set-options ovn-subnet-2-ovn-cluster router-port=ovn-cluster-ovn-subnet-2
// 在debian01上用命令查看北向数据库和南向数据库信息
root@debian01:~# ovn-nbctl show
switch eff58e2c-02b8-4c24-a0f0-72e785b98781 (ovn-subnet-2)
port ovn-subnet-2-ovn-cluster
type: router
addresses: ["00:00:00:00:12:00 192.168.2.1"]
router-port: ovn-cluster-ovn-subnet-2
port ovn-subnet-2-debian02-ns2
addresses: ["00:00:00:00:00:04 192.168.2.30"]
switch d7ab026e-09ed-4ac7-9491-2510caa1c57d (ovn-default)
port ovn-default-debian01-ns1
addresses: ["00:00:00:00:00:01 192.168.1.10"]
port ovn-default-debian03-ns1
addresses: ["00:00:00:00:00:03 192.168.1.30"]
port ovn-default-ovn-cluster
type: router
addresses: ["00:00:00:00:11:00 192.168.1.1"]
router-port: ovn-cluster-ovn-default
port ovn-default-debian02-ns1
addresses: ["00:00:00:00:00:02 192.168.1.20"]
router 168d8fba-fb1d-45c5-a247-9e67dde7bbcd (ovn-cluster)
port ovn-cluster-ovn-default
mac: "00:00:00:00:11:00"
networks: ["192.168.1.1/24"]
port ovn-cluster-ovn-subnet-2
mac: "00:00:00:00:12:00"
networks: ["192.168.2.1/24"]
// 在debian01上给ns1添加默认路由
ip netns exec ns1 ip route add default via 192.168.1.1 dev veth12
// 在debian02上给ns1添加默认路由
ip netns exec ns1 ip route add default via 192.168.1.1 dev veth12
// 在debian02上给ns2添加默认路由
ip netns exec ns2 ip route add default via 192.168.2.1 dev veth22
// 在debian03上给ns3添加默认路由
ip netns exec ns1 ip route add default via 192.168.1.1 dev veth12
// debian01上ns1连通网关
root@debian01:~# ip netns exec ns1 ping -c 1 192.168.1.1
PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
64 bytes from 192.168.1.1: icmp_seq=1 ttl=254 time=0.932 ms
--- 192.168.1.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.932/0.932/0.932/0.000 ms
root@debian02:~# ip netns exec ns2 ping -c 1 192.168.2.1
PING 192.168.2.1 (192.168.2.1) 56(84) bytes of data.
64 bytes from 192.168.2.1: icmp_seq=1 ttl=254 time=1.18 ms
--- 192.168.2.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.177/1.177/1.177/0.000 ms
root@debian01:~# ip netns exec ns1 ping -c 1 192.168.2.30
PING 192.168.2.30 (192.168.2.30) 56(84) bytes of data.
64 bytes from 192.168.2.30: icmp_seq=1 ttl=63 time=2.44 ms
--- 192.168.2.30 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.435/2.435/2.435/0.000 ms
// 在debian01上 添加逻辑交换机及端口
ovn-nbctl ls-add join
ovn-nbctl lsp-add join node-debian01
ovn-nbctl lsp-set-addresses node-debian01 00:00:00:02:00:01
ovn-nbctl lsp-set-port-security node-debian01 "00:00:00:02:00:01 100.64.0.2"
ovn-nbctl lsp-add join node-debian02
ovn-nbctl lsp-set-addresses node-debian02 00:00:00:03:00:01
ovn-nbctl lsp-set-port-security node-debian02 "00:00:00:03:00:01 100.64.0.3"
ovn-nbctl lsp-add join node-debian03
ovn-nbctl lsp-set-addresses node-debian03 00:00:00:04:00:01
ovn-nbctl lsp-set-port-security node-debian03 "00:00:00:04:00:01 100.64.0.4"
// 在debian01上添加路由器端口
ovn-nbctl lrp-add ovn-cluster ovn-cluster-join 00:00:00:00:13:00 100.64.0.1/24
// 在debian01上将交换机与路由器连接
ovn-nbctl lsp-add join join-ovn-cluster-join
ovn-nbctl lsp-set-type join-ovn-cluster-join router
ovn-nbctl lsp-set-addresses join-ovn-cluster-join "00:00:00:00:13:00 100.64.0.1"
ovn-nbctl lsp-set-options join-ovn-cluster-join router-port=ovn-cluster-join
// 在debian01 操作
ovs-vsctl add-port br-int ovn0 -- set interface ovn0 type=internal external_ids:iface-id=node-debian01
ip link set dev ovn0 address 00:00:00:02:00:01
ip addr add 100.64.0.2/24 dev ovn0
ip link set dev ovn0 up
// debian02 操作
ovs-vsctl add-port br-int ovn0 -- set interface ovn0 type=internal external_ids:iface-id=node-debian02
ip link set dev ovn0 address 00:00:00:03:00:01
ip addr add 100.64.0.3/24 dev ovn0
ip link set dev ovn0 up
// debian03 操作
ovs-vsctl add-port br-int ovn0 -- set interface ovn0 type=internal external_ids:iface-id=node-debian03
ip link set dev ovn0 address 00:00:00:04:00:01
ip addr add 100.64.0.4/24 dev ovn0
ip link set dev ovn0 up
// 所有节点添加到容器网络的路由
ip route add 192.168.1.0/24 via 100.64.0.1 dev ovn0
ip route add 192.168.2.0/24 via 100.64.0.1 dev ovn0
// 测试在debian01
root@debian01:~# ping -c 1 100.64.0.1
PING 100.64.0.1 (100.64.0.1) 56(84) bytes of data.
64 bytes from 100.64.0.1: icmp_seq=1 ttl=254 time=5.36 ms
--- 100.64.0.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 5.360/5.360/5.360/0.000 ms
root@debian01:~# ping -c 1 100.64.0.2
PING 100.64.0.2 (100.64.0.2) 56(84) bytes of data.
64 bytes from 100.64.0.2: icmp_seq=1 ttl=64 time=0.152 ms
--- 100.64.0.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.152/0.152/0.152/0.000 ms
root@debian01:~# ping -c 1 100.64.0.3
PING 100.64.0.3 (100.64.0.3) 56(84) bytes of data.
64 bytes from 100.64.0.3: icmp_seq=1 ttl=64 time=3.45 ms
--- 100.64.0.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 3.447/3.447/3.447/0.000 ms
root@debian01:~# ping -c 1 100.64.0.4
PING 100.64.0.4 (100.64.0.4) 56(84) bytes of data.
64 bytes from 100.64.0.4: icmp_seq=1 ttl=64 time=3.08 ms
--- 100.64.0.4 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 3.076/3.076/3.076/0.000 ms
root@debian01:~# ping -c 1 192.168.1.1
PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
64 bytes from 192.168.1.1: icmp_seq=1 ttl=254 time=0.569 ms
--- 192.168.1.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.569/0.569/0.569/0.000 ms
root@debian01:~# ping -c 1 192.168.1.10
PING 192.168.1.10 (192.168.1.10) 56(84) bytes of data.
64 bytes from 192.168.1.10: icmp_seq=1 ttl=63 time=1.01 ms
--- 192.168.1.10 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.013/1.013/1.013/0.000 ms
root@debian01:~# ping -c 1 192.168.1.20
PING 192.168.1.20 (192.168.1.20) 56(84) bytes of data.
64 bytes from 192.168.1.20: icmp_seq=1 ttl=63 time=2.17 ms
--- 192.168.1.20 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.169/2.169/2.169/0.000 ms
root@debian01:~# ping -c 1 192.168.2.30
PING 192.168.2.30 (192.168.2.30) 56(84) bytes of data.
64 bytes from 192.168.2.30: icmp_seq=1 ttl=63 time=2.04 ms
--- 192.168.2.30 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 2.038/2.038/2.038/0.000 ms
root@debian01:~#
ovn-nbctl create load_balancer vips:10.101.0.1="192.168.1.10,192.168.1.20"
ovn-nbctl list load_balancer
ovn-nbctl lb-list
UUID LB PROTO VIP IPs
7b7ffead-34c7-4c0a-898a-32afaceb7f02 10.101.0.1 192.168.1.10,192.168.1.20
ovn-nbctl set logical_router ovn-cluster load_balancer=7b7ffead-34c7-4c0a-898a-32afaceb7f02
ovn-nbctl lr-lb-list ovn-cluster
mark
因为cilium的还有一部份还未写完,coming soon