Closed nics90 closed 4 months ago
vysr2939
commented
7 months ago I am seeing the same issue... @nics90 / @bobz965 is there any workaround you are aware till the bug is fixed? I am trying to attach multiple non default subnets though multus to kubevirt VM and unable to push default route.
nics90
commented
7 months ago @bobz965 : Could you please suggest any workaround till the time fix is available or let me know if any help is required in fixing the bug.
bobz965
commented
7 months ago @bobz965 : Could you please suggest any workaround till the time fix is available or let me know if any help is required in fixing the bug.
I will try to fix it next week.
nics90
commented
7 months ago @bobz965 : Did you get chance to look into this issue ?
bobz965
commented
7 months ago @bobz965 : Did you get chance to look into this issue ?
sorry, not yet. too busy, please wait, thanks
nics90
commented
6 months ago Understand @bobz965, please let us know if I can help anyhow, since this issue is a blocker for us.
bobz965
commented
6 months ago Understand @bobz965, please let us know if I can help anyhow, since this issue is a blocker for us.
ok. thanks
bobz965
commented
6 months ago hi @nics90 , I retest the issue in master, seems no problem
root@empty:~/test1/multus# k get subnet
NAME PROVIDER VPC PROTOCOL CIDR PRIVATE NAT DEFAULT GATEWAYTYPE V4USED V4AVAILABLE V6USED V6AVAILABLE EXCLUDEIPS U2OINTERCONNECTIONIP
join ovn ovn-cluster IPv4 100.64.0.0/16 false false false distributed 2 65531 0 0 ["100.64.0.1"]
overlay-subnet-111315925 ovn vpc-151919753 IPv4 10.0.0.0/24 false false false distributed 5 248 0 0 ["10.0.0.1"]
ovn-default ovn ovn-cluster IPv4 10.16.0.0/16 false true true distributed 4 65529 0 0 ["10.16.0.1"]
ovn-vpc-external-network ovn-vpc-external-network.kube-system IPv4 172.22.0.0/16 false false false distributed 6 65525 0 0 ["172.22.0.1","172.22.0.2","172.22.0.3"]
root@empty:~/test1/multus#
root@empty:~/test1/multus# cat multi-nic.yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
k8s.v1.cni.cncf.io/networks: kube-system/ovn-vpc-external-network
ovn.kubernetes.io/logical_switch: overlay-subnet-111315925 # default network is a custom vpc subnet
name: multi-nic
namespace: kube-system
spec:
affinity: {}
containers:
- args:
- -c
- while true; do sleep 10000; done
command:
- bash
image: docker.io/kubeovn/vpc-nat-gateway:v1.13.0
imagePullPolicy: IfNotPresent
name: vpc-nat-gw
resources: {}
securityContext:
allowPrivilegeEscalation: true
privileged: true
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
root@empty:~/test1/multus#
# after I apply the Yaml, then i get pod which use the custom vpc subnetwork
root@empty:~/test1/multus# k exec -it -n kube-system multi-nic -- bash
multi-nic:/kube-ovn#
multi-nic:/kube-ovn#
multi-nic:/kube-ovn# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host proto kernel_lo
valid_lft forever preferred_lft forever
2: net1@if42: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether da:22:ad:f7:d3:fd brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 172.22.0.9/16 brd 172.22.255.255 scope global net1
valid_lft forever preferred_lft forever
inet6 fe80::d822:adff:fef7:d3fd/64 scope link proto kernel_ll
valid_lft forever preferred_lft forever
63: eth0@if64: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1400 qdisc noqueue state UP group default
link/ether 00:00:00:59:15:37 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.0.0.5/24 brd 10.0.0.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::200:ff:fe59:1537/64 scope link proto kernel_ll
valid_lft forever preferred_lft forever
root@empty:~/test1/multus# k get po -n kube-system multi-nic -o yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
k8s.v1.cni.cncf.io/network-status: |-
[{
"name": "kube-ovn",
"interface": "eth0",
"ips": [
"10.0.0.5"
],
"mac": "00:00:00:59:15:37",
"default": true,
"dns": {},
"gateway": [
"10.0.0.1"
]
},{
"name": "kube-system/ovn-vpc-external-network",
"interface": "net1",
"ips": [
"172.22.0.9"
],
"mac": "da:22:ad:f7:d3:fd",
"dns": {}
}]
k8s.v1.cni.cncf.io/networks: kube-system/ovn-vpc-external-network
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"kube-system/ovn-vpc-external-network","ovn.kubernetes.io/logical_switch":"overlay-subnet-111315925"},"name":"multi-nic","namespace":"kube-system"},"spec":{"affinity":{},"containers":[{"args":["-c","while true; do sleep 10000; done"],"command":["bash"],"image":"docker.io/kubeovn/vpc-nat-gateway:v1.13.0","imagePullPolicy":"IfNotPresent","name":"vpc-nat-gw","resources":{},"securityContext":{"allowPrivilegeEscalation":true,"privileged":true}}],"priority":0,"restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"serviceAccount":"default","serviceAccountName":"default"}}
ovn-vpc-external-network.kube-system.kubernetes.io/allocated: "true"
ovn-vpc-external-network.kube-system.kubernetes.io/cidr: 172.22.0.0/16
ovn-vpc-external-network.kube-system.kubernetes.io/gateway: 172.22.0.1
ovn-vpc-external-network.kube-system.kubernetes.io/ip_address: 172.22.0.9
ovn-vpc-external-network.kube-system.kubernetes.io/mac_address: 00:00:00:0C:3C:D8
ovn.kubernetes.io/allocated: "true"
ovn.kubernetes.io/cidr: 10.0.0.0/24
ovn.kubernetes.io/gateway: 10.0.0.1
ovn.kubernetes.io/ip_address: 10.0.0.5
ovn.kubernetes.io/logical_router: vpc-151919753
ovn.kubernetes.io/logical_switch: overlay-subnet-111315925
ovn.kubernetes.io/mac_address: "00:00:00:59:15:37"
ovn.kubernetes.io/pod_nic_type: veth-pair
ovn.kubernetes.io/routed: "true"
creationTimestamp: "2024-01-30T09:09:18Z"
name: multi-nic
namespace: kube-system
resourceVersion: "4934"
uid: eda05526-e046-4e19-8ea9-a55ecf937a10
spec:
affinity: {}
containers:
- args:
- -c
- while true; do sleep 10000; done
command:
- bash
image: docker.io/kubeovn/vpc-nat-gateway:v1.13.0
imagePullPolicy: IfNotPresent
name: vpc-nat-gw
resources: {}
securityContext:
allowPrivilegeEscalation: true
privileged: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-knzzb
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
nodeName: kube-ovn-worker
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- name: kube-api-access-knzzb
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2024-01-30T09:09:18Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2024-01-30T09:09:20Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2024-01-30T09:09:20Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2024-01-30T09:09:18Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: containerd://435499da2a28131b44ec85eac1ba4df7a9ac4e1eebea7d4f0c38b4d3d40d609d
image: docker.io/kubeovn/vpc-nat-gateway:v1.13.0
imageID: docker.io/library/import-2024-01-30@sha256:c8e03b7371b1610409cb17d7eda0a7fcb011d31b0329c8fd2b68a804002c60c7
lastState: {}
name: vpc-nat-gw
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2024-01-30T09:09:19Z"
hostIP: 172.18.0.3
phase: Running
podIP: 10.0.0.5
podIPs:
- ip: 10.0.0.5
qosClass: BestEffort
startTime: "2024-01-30T09:09:18Z"
can you retest my example pod yaml in your env, maybe pod annos for multi nic is not right? I still not test this in release 1.12.
nics90
commented
6 months ago @bobz965 : Is there any dependency of kube ovn v1.12.3 on multus CNI version, since we are using multus version v3.8 [sha: 9479537fe0827d23bc40056e98f8d1e75778ec294d89ae4d8a62f83dfc74a31d] ?
Also, we found that in Makefile: MULTUS_VERSION = v4.0.2
bobz965
commented
6 months ago @bobz965 : Is there any dependency of kube ovn v1.12.3 on multus CNI version, since we are using multus version v3.8 [sha: 9479537fe0827d23bc40056e98f8d1e75778ec294d89ae4d8a62f83dfc74a31d] ?
Also, we found that in Makefile: MULTUS_VERSION = v4.0.2
I'm not clear about this, please try MULTUS_VERSION = v4.0.2
zcq98
commented
6 months ago try setting annotation in vm： v1.multus-cni.io/default-network: vm-registry/test-default-98fl8 ovn.kubernetes.io/logical_switch: test-default-sub
Ultima1252
commented
6 months ago One other thing about this bug that I have noticed. It seems that only the 'thick' version of multus is affected. I switched to the non-thick daemon and don't seem to be hitting this issue anymore.
zhangzujian
commented
4 months ago Please upgrade kube-ovn to the latest v1.12.x version.
Bug Report
While creating kubevirt VM, when we add network of type multus, we are unable to set "vmi.spec.network.multus" to true when network-attachment-definition is created of type kubeovn. This is the only way to control default network when multiple networks are attached to VM and also to assign DHCP ip with with proper gateway
This issue is being seen in all 1.12.x versions. However it works perfectly fine in 1.11.x versions
Expected Behavior
virt-launcher pod should be created with a corresponding vswitch in the annotation and IP of the virt-launcher pod and vmi should be assigned from the custom subnet & Network Attachment Definition
Actual Behavior
VMI and virt-launcher is getting created in "ovn-default" subnet
Steps to Reproduce the Problem
Additional Info
After the VM creation VMI and virt-launcher pod gets wrong IP from ovn-default instead of “test-default-sub”
virt-launcher pod has no annotation related to vswitch that corresponds to the subnet created above (full yaml is attached)
Kubernetes version:
Output of
kubectl version:kube-ovn version:
operation-system/kernel version:
Output of
awk -F '=' '/PRETTY_NAME/ { print $2 }' /etc/os-release: Output ofuname -r:Linux k8s-devmaster01 5.4.0-169-generic #187-Ubuntu SMP Thu Nov 23 14:52:28 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux