feature: fixed ip for kubevirt

[weizhoublue]

vmi and its vm pod

VM pod

~# kubectl get vmi vm-bridge-calico -o yaml
apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
  annotations:
    kubevirt.io/latest-observed-api-version: v1
    kubevirt.io/storage-observed-api-version: v1
    kubevirt.io/vm-generation: "3"
  creationTimestamp: "2023-08-08T07:00:18Z"
  finalizers:
  - kubevirt.io/virtualMachineControllerFinalize
  - foregroundDeleteVirtualMachine
  generation: 9
  labels:
    kubevirt.io/nodeName: 10-20-1-20
  name: vm-bridge-calico
  namespace: default
  ownerReferences:
  - apiVersion: kubevirt.io/v1
    blockOwnerDeletion: true
    controller: true
    kind: VirtualMachine
    name: vm-bridge-calico
    uid: e065d3e8-4b16-46c3-ac06-7321209c3da2
  resourceVersion: "25576607"
  uid: 7c21d9c6-6247-4adf-a905-e3055afb93ba
spec:
  architecture: amd64
  domain:
    cpu:
      cores: 1
      model: host-model
      sockets: 1
      threads: 1
    devices:
      disks:
      - disk:
          bus: virtio
        name: containerdisk
      - disk:
          bus: virtio
        name: cloudinitdisk
      interfaces:
      - bridge: {}
        name: red
      rng: {}
    features:
      acpi:
        enabled: true
    firmware:
      uuid: 8af52f42-27f9-5ddd-a336-bf679199d633
    machine:
      type: q35
    resources:
      requests:
        memory: 1024M
  networks:
  - name: red
    pod: {}
  terminationGracePeriodSeconds: 0
  volumes:
  - containerDisk:
      image: kubevirt/fedora-with-test-tooling-container-disk:devel
      imagePullPolicy: IfNotPresent
    name: containerdisk
  - cloudInitNoCloud:
      userData: |-
        #!/bin/bash
        echo "fedora" |passwd fedora --stdin
    name: cloudinitdisk
status:
  activePods:
    59faca5b-987c-49dc-b9c7-e9bd086128d0: 10-20-1-20
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2023-08-08T07:00:28Z"
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: null
    message: cannot migrate VMI which does not use masquerade to connect to the pod
      network or bridge with kubevirt.io/allow-pod-bridge-network-live-migration VM
      annotation
    reason: InterfaceNotLiveMigratable
    status: "False"
    type: LiveMigratable
  currentCPUTopology:
    cores: 1
    sockets: 1
    threads: 1
  guestOSInfo: {}
  interfaces:
  - infoSource: domain
    ipAddress: 172.21.240.33
    ipAddresses:
    - 172.21.240.33
    - fc33::3d68:dc5a:8181:d66b
    mac: b6:df:8d:9c:10:99
    name: red
    queueCount: 1
  launcherContainerImageVersion: quay.m.daocloud.io/kubevirt/virt-launcher:v1.0.0
  machine:
    type: pc-q35-rhel9.2.0
  migrationMethod: BlockMigration
  migrationTransport: Unix
  nodeName: 10-20-1-20
  phase: Running
  phaseTransitionTimestamps:
  - phase: Pending
    phaseTransitionTimestamp: "2023-08-08T07:00:18Z"
  - phase: Scheduling
    phaseTransitionTimestamp: "2023-08-08T07:00:18Z"
  - phase: Scheduled
    phaseTransitionTimestamp: "2023-08-08T07:00:28Z"
  - phase: Running
    phaseTransitionTimestamp: "2023-08-08T07:00:30Z"
  qosClass: Burstable
  runtimeUser: 107
  selinuxContext: none
  virtualMachineRevisionName: revision-start-vm-e065d3e8-4b16-46c3-ac06-7321209c3da2-3
  volumeStatus:
  - name: cloudinitdisk
    size: 1048576
    target: vdb
  - name: containerdisk
    target: vda

~# kubectl get pod virt-launcher-vm-bridge-calico-dn7zf -o yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    cni.projectcalico.org/containerID: a118ea63f23f889df73fcfeb5a754934ffdc1103d178d525fe647a91c048c02a
    cni.projectcalico.org/podIP: 172.21.240.33/32
    cni.projectcalico.org/podIPs: 172.21.240.33/32,fc33::3d68:dc5a:8181:d66b/128
    k8s.v1.cni.cncf.io/network-status: |-
      [{
          "name": "calico-pod-network",
          "ips": [
              "172.21.240.33",
              "fc33::3d68:dc5a:8181:d66b"
          ],
          "default": true,
          "dns": {}
      }]
    kubectl.kubernetes.io/default-container: compute
    kubevirt.io/domain: vm-bridge-calico
    kubevirt.io/migrationTransportUnix: "true"
    kubevirt.io/vm-generation: "3"
    post.hook.backup.velero.io/command: '["/usr/bin/virt-freezer", "--unfreeze", "--name",
      "vm-bridge-calico", "--namespace", "default"]'
    post.hook.backup.velero.io/container: compute
    pre.hook.backup.velero.io/command: '["/usr/bin/virt-freezer", "--freeze", "--name",
      "vm-bridge-calico", "--namespace", "default"]'
    pre.hook.backup.velero.io/container: compute
  creationTimestamp: "2023-08-08T07:00:18Z"
  generateName: virt-launcher-vm-bridge-calico-
  labels:
    kubevirt.io: virt-launcher
    kubevirt.io/created-by: 7c21d9c6-6247-4adf-a905-e3055afb93ba
    kubevirt.io/nodeName: 10-20-1-20
    vm.kubevirt.io/name: vm-bridge-calico
  name: virt-launcher-vm-bridge-calico-dn7zf
  namespace: default
  ownerReferences:
  - apiVersion: kubevirt.io/v1
    blockOwnerDeletion: true
    controller: true
    kind: VirtualMachineInstance
    name: vm-bridge-calico
    uid: 7c21d9c6-6247-4adf-a905-e3055afb93ba
  resourceVersion: "25576587"
  uid: 59faca5b-987c-49dc-b9c7-e9bd086128d0
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: node-labeller.kubevirt.io/obsolete-host-model
            operator: DoesNotExist
  automountServiceAccountToken: false
  containers:
  - command:
    - /usr/bin/virt-launcher-monitor
    - --qemu-timeout
    - 329s
    - --name
    - vm-bridge-calico
    - --uid
    - 7c21d9c6-6247-4adf-a905-e3055afb93ba
    - --namespace
    - default
    - --kubevirt-share-dir
    - /var/run/kubevirt
    - --ephemeral-disk-dir
    - /var/run/kubevirt-ephemeral-disks
    - --container-disk-dir
    - /var/run/kubevirt/container-disks
    - --grace-period-seconds
    - "15"
    - --hook-sidecars
    - "0"
    - --ovmf-path
    - /usr/share/OVMF
    - --run-as-nonroot
    env:
    - name: XDG_CACHE_HOME
      value: /var/run/kubevirt-private
    - name: XDG_CONFIG_HOME
      value: /var/run/kubevirt-private
    - name: XDG_RUNTIME_DIR
      value: /var/run
    - name: POD_NAME
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: metadata.name
    image: quay.m.daocloud.io/kubevirt/virt-launcher:v1.0.0
    imagePullPolicy: IfNotPresent
    name: compute
    resources:
      limits:
        devices.kubevirt.io/kvm: "1"
        devices.kubevirt.io/tun: "1"
        devices.kubevirt.io/vhost-net: "1"
      requests:
        cpu: 100m
        devices.kubevirt.io/kvm: "1"
        devices.kubevirt.io/tun: "1"
        devices.kubevirt.io/vhost-net: "1"
        ephemeral-storage: 50M
        memory: "1279755392"
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        add:
        - NET_BIND_SERVICE
        drop:
        - ALL
      privileged: false
      runAsGroup: 107
      runAsNonRoot: true
      runAsUser: 107
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /var/run/kubevirt-private
      name: private
    - mountPath: /var/run/kubevirt
      name: public
    - mountPath: /var/run/kubevirt-ephemeral-disks
      name: ephemeral-disks
    - mountPath: /var/run/kubevirt/container-disks
      mountPropagation: HostToContainer
      name: container-disks
    - mountPath: /var/run/libvirt
      name: libvirt-runtime
    - mountPath: /var/run/kubevirt/sockets
      name: sockets
    - mountPath: /var/run/kubevirt/hotplug-disks
      mountPropagation: HostToContainer
      name: hotplug-disks
  - args:
    - --copy-path
    - /var/run/kubevirt-ephemeral-disks/container-disk-data/7c21d9c6-6247-4adf-a905-e3055afb93ba/disk_0
    command:
    - /usr/bin/container-disk
    image: kubevirt/fedora-with-test-tooling-container-disk:devel
    imagePullPolicy: IfNotPresent
    name: volumecontainerdisk
    resources:
      limits:
        cpu: 10m
        memory: 40M
      requests:
        cpu: 1m
        ephemeral-storage: 50M
        memory: 1M
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      runAsNonRoot: true
      runAsUser: 107
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /var/run/kubevirt-ephemeral-disks/container-disk-data/7c21d9c6-6247-4adf-a905-e3055afb93ba
      name: container-disks
    - mountPath: /usr/bin
      name: virt-bin-share-dir
  dnsPolicy: ClusterFirst
  enableServiceLinks: false
  hostname: vm-bridge-calico
  initContainers:
  - command:
    - /usr/bin/cp
    - /usr/bin/container-disk
    - /init/usr/bin/container-disk
    env:
    - name: XDG_CACHE_HOME
      value: /var/run/kubevirt-private
    - name: XDG_CONFIG_HOME
      value: /var/run/kubevirt-private
    - name: XDG_RUNTIME_DIR
      value: /var/run
    image: quay.m.daocloud.io/kubevirt/virt-launcher:v1.0.0
    imagePullPolicy: IfNotPresent
    name: container-disk-binary
    resources:
      limits:
        cpu: 100m
        memory: 40M
      requests:
        cpu: 10m
        memory: 1M
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      privileged: false
      runAsGroup: 107
      runAsNonRoot: true
      runAsUser: 107
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /init/usr/bin
      name: virt-bin-share-dir
  - args:
    - --no-op
    command:
    - /usr/bin/container-disk
    image: kubevirt/fedora-with-test-tooling-container-disk:devel
    imagePullPolicy: IfNotPresent
    name: volumecontainerdisk-init
    resources:
      limits:
        cpu: 10m
        memory: 40M
      requests:
        cpu: 1m
        ephemeral-storage: 50M
        memory: 1M
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      runAsNonRoot: true
      runAsUser: 107
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /var/run/kubevirt-ephemeral-disks/container-disk-data/7c21d9c6-6247-4adf-a905-e3055afb93ba
      name: container-disks
    - mountPath: /usr/bin
      name: virt-bin-share-dir
  nodeName: 10-20-1-20
  nodeSelector:
    kubernetes.io/arch: amd64
    kubevirt.io/schedulable: "true"
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  readinessGates:
  - conditionType: kubevirt.io/virtual-machine-unpaused
  restartPolicy: Never
  schedulerName: default-scheduler
  securityContext:
    fsGroup: 107
    runAsGroup: 107
    runAsNonRoot: true
    runAsUser: 107
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  volumes:
  - emptyDir: {}
    name: private
  - emptyDir: {}
    name: public
  - emptyDir: {}
    name: sockets
  - emptyDir: {}
    name: virt-bin-share-dir
  - emptyDir: {}
    name: libvirt-runtime
  - emptyDir: {}
    name: ephemeral-disks
  - emptyDir: {}
    name: container-disks
  - emptyDir: {}
    name: hotplug-disks
status:
  conditions:
  - lastProbeTime: "2023-08-08T07:00:18Z"
    lastTransitionTime: "2023-08-08T07:00:18Z"
    message: the virtual machine is not paused
    reason: NotPaused
    status: "True"
    type: kubevirt.io/virtual-machine-unpaused
  - lastProbeTime: null
    lastTransitionTime: "2023-08-08T07:00:26Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2023-08-08T07:00:28Z"
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2023-08-08T07:00:28Z"
    status: "True"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2023-08-08T07:00:18Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: containerd://d526fc4632718614bb64be52f8be8d9d4bcbff1afd155eb0c22772ea8772ab6a
    image: quay.m.daocloud.io/kubevirt/virt-launcher:v1.0.0
    imageID: quay.m.daocloud.io/kubevirt/virt-launcher@sha256:b9b04d9d06ada4486e6acaa08e99aaa6a050e1843624ba4f291f8d167ebec36b
    lastState: {}
    name: compute
    ready: true
    restartCount: 0
    started: true
    state:
      running:
        startedAt: "2023-08-08T07:00:26Z"
  - containerID: containerd://fe58f364f1e0a53045d815ac833e47d82df75ac876855ca419e1a43246022d95
    image: docker.io/kubevirt/fedora-with-test-tooling-container-disk:devel
    imageID: docker.io/kubevirt/fedora-with-test-tooling-container-disk@sha256:cf045994c8e26e19de998b1c852dd4e7057bf80923449461357be7bbec0b2837
    lastState: {}
    name: volumecontainerdisk
    ready: true
    restartCount: 0
    started: true
    state:
      running:
        startedAt: "2023-08-08T07:00:27Z"
  hostIP: 172.81.0.20
  initContainerStatuses:
  - containerID: containerd://3319fd6ae879102bbe3499c2c5bc01c000771830bfc2811cdfca7bcb68bfb8e1
    image: quay.m.daocloud.io/kubevirt/virt-launcher:v1.0.0
    imageID: quay.m.daocloud.io/kubevirt/virt-launcher@sha256:b9b04d9d06ada4486e6acaa08e99aaa6a050e1843624ba4f291f8d167ebec36b
    lastState: {}
    name: container-disk-binary
    ready: true
    restartCount: 0
    state:
      terminated:
        containerID: containerd://3319fd6ae879102bbe3499c2c5bc01c000771830bfc2811cdfca7bcb68bfb8e1
        exitCode: 0
        finishedAt: "2023-08-08T07:00:23Z"
        reason: Completed
        startedAt: "2023-08-08T07:00:23Z"
  - containerID: containerd://79d2084c6d7b56551136930912b25da3589a992497ffec6da987d9d3b5f8971f
    image: docker.io/kubevirt/fedora-with-test-tooling-container-disk:devel
    imageID: docker.io/kubevirt/fedora-with-test-tooling-container-disk@sha256:cf045994c8e26e19de998b1c852dd4e7057bf80923449461357be7bbec0b2837
    lastState: {}
    name: volumecontainerdisk-init
    ready: true
    restartCount: 0
    state:
      terminated:
        containerID: containerd://79d2084c6d7b56551136930912b25da3589a992497ffec6da987d9d3b5f8971f
        exitCode: 0
        finishedAt: "2023-08-08T07:00:25Z"
        reason: Completed
        startedAt: "2023-08-08T07:00:24Z"
  phase: Running
  podIP: 172.21.240.33
  podIPs:
  - ip: 172.21.240.33
  - ip: fc33::3d68:dc5a:8181:d66b
  qosClass: Burstable
  startTime: "2023-08-08T07:00:18Z"

restart vm

the vmi

status:
  activePods:
    1a3b0760-b115-4b05-aee1-b40fef9a6f36: 10-20-1-20
  phase: Running

status:
  activePods:
    1a3b0760-b115-4b05-aee1-b40fef9a6f36: 10-20-1-20
  phase: Succeeded

status:
  activePods:
    d4df4aea-f2be-42ed-a78c-dfc9ce173239: 10-20-1-20
  phase: Running

思路： endpoint: (1) kubevirt: true (2) vmi: uuid (3) pod: uuid

list 监控 VMI 删除事件，是否 ippool 中的 IP

[weizhoublue]

热迁移前后

迁移中
    ~# kubectl get VirtualMachineInstanceMigration -o wide
      NAME                        PHASE        VMI
      kubevirt-migrate-vm-6wgr7   Scheduling   vm-migration

    ~# kubectl get vm
      NAME                      AGE     STATUS    READY
      vm-migration              117s    Running   True

    ~# kubectl get vmi
      NAME                      AGE     PHASE     IP              NODENAME          READY
      vm-migration              113s    Running   172.21.175.14   kubevirt-worker   True

    ~# kubectl get pod -o wide
    NAME                                          READY   STATUS     RESTARTS        AGE
    virt-launcher-vm-migration-lv7h8              2/2     Running    0               2m15s   172.21.175.14   kubevirt-worker          <none>           1/1
    virt-launcher-vm-migration-rmqvm              0/2     Init:1/2   0               52s     172.21.18.159   kubevirt-control-plane   <none>           0/1

迁移接近完成时，两个pod 都是 running 

迁移完成
    ~# kubectl get VirtualMachineInstanceMigration -o wide
      NAME                        PHASE       VMI
      kubevirt-migrate-vm-swckv   Succeeded   vm-migration

    ~# kubectl get vmi
      NAME                      AGE     PHASE     IP              NODENAME                 READY
      vm-migration              147m    Running   172.21.18.160   kubevirt-control-plane   True

    老 pod 一致处于 Completed 状态
    ~# kubectl get pod -o wide
      NAME                                           READY   STATUS      RESTARTS     AGE     IP              NODE                     NOMINATED NODE   READINESS GATES
      virt-launcher-vm-migration-g89xw               2/2     Running     0            4h6m    172.21.18.160   kubevirt-control-plane   <none>           1/1
      virt-launcher-vm-migration-lv7h8               0/2     Completed   0            4h44m   172.21.175.14   kubevirt-worker          <none>           1/1

问题：在迁移过程中，2个pod 都是同时 running 的时间，分配固定 IP 就要在 老pod 还未 Completed 就把 IP 分配新POD使用，这样短暂瞬间 是否会出问题 ？ 并且 这个瞬间 会因为 新pod arping 失败 而 启动失败，如何解决 ？

固定IP 支持 热迁移 比较麻烦