search

kubeovn / kube-ovn

A Bridge between SDN and Cloud Native (Project under CNCF)
https://kubeovn.github.io/docs/stable/en/
Apache License 2.0
1.87k stars 433 forks source link

[Bug] Use JSON merge patch to update annotations #4229

Open zhangzujian opened 1 week ago

zhangzujian commented 1 week ago

Description

Currently, we are using JSON patch to add/update annotations, this may cause an unexpected result.

Here is an example of the annotation ovn.kubernetes.io/chassis was removed by kube-ovn-controller:

[
    {
        "kind": "Event",
        "apiVersion": "audit.k8s.io/v1",
        "level": "RequestResponse",
        "auditID": "b3ec7bb5-f2bc-4bc5-b67e-f05ae8ef4da0",
        "stage": "ResponseComplete",
        "requestURI": "/api/v1/nodes/kube-ovn-control-plane",
        "verb": "patch",
        "user": {
            "username": "system:serviceaccount:kube-system:kube-ovn-cni",
            "uid": "7732b00e-e2a6-4aca-b9ce-abcc4ad6863b",
            "groups": [
                "system:serviceaccounts",
                "system:serviceaccounts:kube-system",
                "system:authenticated"
            ],
            "extra": {
                "authentication.kubernetes.io/pod-name": [
                    "kube-ovn-cni-7hzmp"
                ],
                "authentication.kubernetes.io/pod-uid": [
                    "48818ed7-149f-46bf-ad0c-addc3d6fe67f"
                ]
            }
        },
        "sourceIPs": [
            "172.18.0.2"
        ],
        "userAgent": "kube-ovn-daemon/v0.0.0 (linux/amd64) kubernetes/$Format",
        "objectRef": {
            "resource": "nodes",
            "name": "kube-ovn-control-plane",
            "apiVersion": "v1"
        },
        "responseStatus": {
            "metadata": {},
            "code": 200
        },
        "requestObject": [
            {
                "op": "add",
                "path": "/metadata/annotations",
                "value": {
                    "kubeadm.alpha.kubernetes.io/cri-socket": "unix:///run/containerd/containerd.sock",
                    "node.alpha.kubernetes.io/ttl": "0",
                    "ovn.kubernetes.io/chassis": "e004137c-a176-49dd-afb6-061569229837",
                    "volumes.kubernetes.io/controller-managed-attach-detach": "true"
                }
            }
        ],
        "responseObject": {
            "kind": "Node",
            "apiVersion": "v1",
            "metadata": {
                "name": "kube-ovn-control-plane",
                "uid": "9f4c73dc-b310-45f8-8dfd-533f86c6a154",
                "resourceVersion": "790",
                "creationTimestamp": "2024-06-27T02:10:42Z",
                "labels": {
                    "beta.kubernetes.io/arch": "amd64",
                    "beta.kubernetes.io/os": "linux",
                    "kube-ovn/role": "master",
                    "kubernetes.io/arch": "amd64",
                    "kubernetes.io/hostname": "kube-ovn-control-plane",
                    "kubernetes.io/os": "linux",
                    "node-role.kubernetes.io/control-plane": "",
                    "node.kubernetes.io/exclude-from-external-load-balancers": "",
                    "type": "kind"
                },
                "annotations": {
                    "kubeadm.alpha.kubernetes.io/cri-socket": "unix:///run/containerd/containerd.sock",
                    "node.alpha.kubernetes.io/ttl": "0",
                    "ovn.kubernetes.io/chassis": "e004137c-a176-49dd-afb6-061569229837",
                    "volumes.kubernetes.io/controller-managed-attach-detach": "true"
                }
            }
        },
        "requestReceivedTimestamp": "2024-06-27T02:11:37.896404Z",
        "stageTimestamp": "2024-06-27T02:11:37.900662Z"
    },
    {
        "kind": "Event",
        "apiVersion": "audit.k8s.io/v1",
        "level": "RequestResponse",
        "auditID": "ab4bebfa-7737-4bf0-8589-082b1ea00497",
        "stage": "ResponseComplete",
        "requestURI": "/api/v1/nodes/kube-ovn-control-plane?timeout=30s",
        "verb": "patch",
        "user": {
            "username": "system:serviceaccount:kube-system:ovn",
            "uid": "7b2a6b93-678c-42d5-bfdb-b0ed9a9bf756",
            "groups": [
                "system:serviceaccounts",
                "system:serviceaccounts:kube-system",
                "system:authenticated"
            ],
            "extra": {
                "authentication.kubernetes.io/pod-name": [
                    "kube-ovn-controller-65956cd878-2hsgk"
                ],
                "authentication.kubernetes.io/pod-uid": [
                    "e1469a38-c5cf-46fb-8567-c4290c7b26c8"
                ]
            }
        },
        "sourceIPs": [
            "172.18.0.2"
        ],
        "userAgent": "kube-ovn-controller/v0.0.0 (linux/amd64) kubernetes/$Format",
        "objectRef": {
            "resource": "nodes",
            "name": "kube-ovn-control-plane",
            "apiVersion": "v1"
        },
        "responseStatus": {
            "metadata": {},
            "code": 200
        },
        "requestObject": [
            {
                "op": "replace",
                "path": "/metadata/annotations",
                "value": {
                    "kubeadm.alpha.kubernetes.io/cri-socket": "unix:///run/containerd/containerd.sock",
                    "node.alpha.kubernetes.io/ttl": "0",
                    "ovn.kubernetes.io/allocated": "true",
                    "ovn.kubernetes.io/cidr": "100.64.0.0/16,fd00:100:64::/112",
                    "ovn.kubernetes.io/gateway": "100.64.0.1,fd00:100:64::1",
                    "ovn.kubernetes.io/ip_address": "100.64.0.2,fd00:100:64::2",
                    "ovn.kubernetes.io/logical_switch": "join",
                    "ovn.kubernetes.io/mac_address": "5a:87:97:ee:ca:7c",
                    "ovn.kubernetes.io/port_name": "node-kube-ovn-control-plane",
                    "volumes.kubernetes.io/controller-managed-attach-detach": "true"
                }
            }
        ],
        "responseObject": {
            "kind": "Node",
            "apiVersion": "v1",
            "metadata": {
                "name": "kube-ovn-control-plane",
                "uid": "9f4c73dc-b310-45f8-8dfd-533f86c6a154",
                "resourceVersion": "792",
                "creationTimestamp": "2024-06-27T02:10:42Z",
                "labels": {
                    "beta.kubernetes.io/arch": "amd64",
                    "beta.kubernetes.io/os": "linux",
                    "kube-ovn/role": "master",
                    "kubernetes.io/arch": "amd64",
                    "kubernetes.io/hostname": "kube-ovn-control-plane",
                    "kubernetes.io/os": "linux",
                    "node-role.kubernetes.io/control-plane": "",
                    "node.kubernetes.io/exclude-from-external-load-balancers": "",
                    "type": "kind"
                },
                "annotations": {
                    "kubeadm.alpha.kubernetes.io/cri-socket": "unix:///run/containerd/containerd.sock",
                    "node.alpha.kubernetes.io/ttl": "0",
                    "ovn.kubernetes.io/allocated": "true",
                    "ovn.kubernetes.io/cidr": "100.64.0.0/16,fd00:100:64::/112",
                    "ovn.kubernetes.io/gateway": "100.64.0.1,fd00:100:64::1",
                    "ovn.kubernetes.io/ip_address": "100.64.0.2,fd00:100:64::2",
                    "ovn.kubernetes.io/logical_switch": "join",
                    "ovn.kubernetes.io/mac_address": "5a:87:97:ee:ca:7c",
                    "ovn.kubernetes.io/port_name": "node-kube-ovn-control-plane",
                    "volumes.kubernetes.io/controller-managed-attach-detach": "true"
                }
            }
        },
        "requestReceivedTimestamp": "2024-06-27T02:11:37.926589Z",
        "stageTimestamp": "2024-06-27T02:11:37.930816Z"
    }
]

Who will benefit from this feature?

No response

Anything else?

No response

zhangzujian commented 1 week ago

We should use JSON merge patch to add/update resource labels/annotations, just like kubectl does:

$ kubectl annotate --overwrite svc kubernetes foo=xyz -v=8
I0627 03:54:37.791236  116852 loader.go:395] Config loaded from file:  /home/zhang/.kube/config
I0627 03:54:37.794171  116852 round_trippers.go:463] GET https://127.0.0.1:34887/api/v1/namespaces/default/services/kubernetes
I0627 03:54:37.794182  116852 round_trippers.go:469] Request Headers:
I0627 03:54:37.794187  116852 round_trippers.go:473]     Accept: application/json
I0627 03:54:37.794190  116852 round_trippers.go:473]     User-Agent: kubectl/v1.30.2 (linux/amd64) kubernetes/3968350
I0627 03:54:37.799188  116852 round_trippers.go:574] Response Status: 200 OK in 4 milliseconds
I0627 03:54:37.799197  116852 round_trippers.go:577] Response Headers:
I0627 03:54:37.799200  116852 round_trippers.go:580]     Cache-Control: no-cache, private
I0627 03:54:37.799202  116852 round_trippers.go:580]     Content-Type: application/json
I0627 03:54:37.799204  116852 round_trippers.go:580]     X-Kubernetes-Pf-Flowschema-Uid: ab36eba1-08ff-4a8a-bf6f-714d794ea8a3
I0627 03:54:37.799206  116852 round_trippers.go:580]     X-Kubernetes-Pf-Prioritylevel-Uid: 0626e4f8-723a-400f-a19d-528c77df29f4
I0627 03:54:37.799208  116852 round_trippers.go:580]     Content-Length: 1233
I0627 03:54:37.799209  116852 round_trippers.go:580]     Date: Thu, 27 Jun 2024 03:54:37 GMT
I0627 03:54:37.799211  116852 round_trippers.go:580]     Audit-Id: b79ad946-2e95-4adb-bba5-592a1543f7ee
I0627 03:54:37.799224  116852 request.go:1212] Response Body: {"kind":"Service","apiVersion":"v1","metadata":{"name":"kubernetes","namespace":"default","uid":"692799b5-7532-4291-bcb6-d69dccf13def","resourceVersion":"478","creationTimestamp":"2024-06-27T03:52:42Z","labels":{"component":"apiserver","provider":"kubernetes"},"annotations":{"foo":"bar"},"managedFields":[{"manager":"kube-apiserver","operation":"Update","apiVersion":"v1","time":"2024-06-27T03:52:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{".":{},"f:component":{},"f:provider":{}}},"f:spec":{"f:clusterIP":{},"f:internalTrafficPolicy":{},"f:ipFamilyPolicy":{},"f:ports":{".":{},"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:sessionAffinity":{},"f:type":{}}}},{"manager":"kubectl-annotate","operation":"Update","apiVersion":"v1","time":"2024-06-27T03:54:21Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:foo":{}}}}}]},"spec":{"ports":[{"name":"https","protocol":"TCP","port":443,"targetPort":6443}],"clu [truncated 209 chars]
I0627 03:54:37.799479  116852 request.go:1212] Request Body: {"metadata":{"annotations":{"foo":"xyz"}}}
I0627 03:54:37.799512  116852 round_trippers.go:463] PATCH https://127.0.0.1:34887/api/v1/namespaces/default/services/kubernetes?fieldManager=kubectl-annotate
I0627 03:54:37.799515  116852 round_trippers.go:469] Request Headers:
I0627 03:54:37.799518  116852 round_trippers.go:473]     Accept: application/json
I0627 03:54:37.799520  116852 round_trippers.go:473]     Content-Type: application/merge-patch+json
I0627 03:54:37.799521  116852 round_trippers.go:473]     User-Agent: kubectl/v1.30.2 (linux/amd64) kubernetes/3968350
I0627 03:54:37.801897  116852 round_trippers.go:574] Response Status: 200 OK in 2 milliseconds
I0627 03:54:37.801915  116852 round_trippers.go:577] Response Headers:
I0627 03:54:37.801921  116852 round_trippers.go:580]     Content-Length: 1233
I0627 03:54:37.801923  116852 round_trippers.go:580]     Date: Thu, 27 Jun 2024 03:54:37 GMT
I0627 03:54:37.801925  116852 round_trippers.go:580]     Audit-Id: 2e3ae111-8ed7-4a5a-a85e-159666f3293b
I0627 03:54:37.801927  116852 round_trippers.go:580]     Cache-Control: no-cache, private
I0627 03:54:37.801929  116852 round_trippers.go:580]     Content-Type: application/json
I0627 03:54:37.801930  116852 round_trippers.go:580]     X-Kubernetes-Pf-Flowschema-Uid: ab36eba1-08ff-4a8a-bf6f-714d794ea8a3
I0627 03:54:37.801932  116852 round_trippers.go:580]     X-Kubernetes-Pf-Prioritylevel-Uid: 0626e4f8-723a-400f-a19d-528c77df29f4
I0627 03:54:37.801955  116852 request.go:1212] Response Body: {"kind":"Service","apiVersion":"v1","metadata":{"name":"kubernetes","namespace":"default","uid":"692799b5-7532-4291-bcb6-d69dccf13def","resourceVersion":"501","creationTimestamp":"2024-06-27T03:52:42Z","labels":{"component":"apiserver","provider":"kubernetes"},"annotations":{"foo":"xyz"},"managedFields":[{"manager":"kube-apiserver","operation":"Update","apiVersion":"v1","time":"2024-06-27T03:52:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{".":{},"f:component":{},"f:provider":{}}},"f:spec":{"f:clusterIP":{},"f:internalTrafficPolicy":{},"f:ipFamilyPolicy":{},"f:ports":{".":{},"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:sessionAffinity":{},"f:type":{}}}},{"manager":"kubectl-annotate","operation":"Update","apiVersion":"v1","time":"2024-06-27T03:54:37Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:foo":{}}}}}]},"spec":{"ports":[{"name":"https","protocol":"TCP","port":443,"targetPort":6443}],"clu [truncated 209 chars]
service/kubernetes annotated

$ kubectl annotate --overwrite svc kubernetes foo- -v=8
I0627 03:55:04.013732  116995 loader.go:395] Config loaded from file:  /home/zhang/.kube/config
I0627 03:55:04.016391  116995 round_trippers.go:463] GET https://127.0.0.1:34887/api/v1/namespaces/default/services/kubernetes
I0627 03:55:04.016407  116995 round_trippers.go:469] Request Headers:
I0627 03:55:04.016411  116995 round_trippers.go:473]     Accept: application/json
I0627 03:55:04.016414  116995 round_trippers.go:473]     User-Agent: kubectl/v1.30.2 (linux/amd64) kubernetes/3968350
I0627 03:55:04.020695  116995 round_trippers.go:574] Response Status: 200 OK in 4 milliseconds
I0627 03:55:04.020708  116995 round_trippers.go:577] Response Headers:
I0627 03:55:04.020711  116995 round_trippers.go:580]     Content-Length: 1233
I0627 03:55:04.020714  116995 round_trippers.go:580]     Date: Thu, 27 Jun 2024 03:55:04 GMT
I0627 03:55:04.020716  116995 round_trippers.go:580]     Audit-Id: 68e55780-2eda-4aea-97b1-ec0eedfb2c20
I0627 03:55:04.020717  116995 round_trippers.go:580]     Cache-Control: no-cache, private
I0627 03:55:04.020719  116995 round_trippers.go:580]     Content-Type: application/json
I0627 03:55:04.020720  116995 round_trippers.go:580]     X-Kubernetes-Pf-Flowschema-Uid: ab36eba1-08ff-4a8a-bf6f-714d794ea8a3
I0627 03:55:04.020722  116995 round_trippers.go:580]     X-Kubernetes-Pf-Prioritylevel-Uid: 0626e4f8-723a-400f-a19d-528c77df29f4
I0627 03:55:04.020740  116995 request.go:1212] Response Body: {"kind":"Service","apiVersion":"v1","metadata":{"name":"kubernetes","namespace":"default","uid":"692799b5-7532-4291-bcb6-d69dccf13def","resourceVersion":"501","creationTimestamp":"2024-06-27T03:52:42Z","labels":{"component":"apiserver","provider":"kubernetes"},"annotations":{"foo":"xyz"},"managedFields":[{"manager":"kube-apiserver","operation":"Update","apiVersion":"v1","time":"2024-06-27T03:52:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{".":{},"f:component":{},"f:provider":{}}},"f:spec":{"f:clusterIP":{},"f:internalTrafficPolicy":{},"f:ipFamilyPolicy":{},"f:ports":{".":{},"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:sessionAffinity":{},"f:type":{}}}},{"manager":"kubectl-annotate","operation":"Update","apiVersion":"v1","time":"2024-06-27T03:54:37Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:foo":{}}}}}]},"spec":{"ports":[{"name":"https","protocol":"TCP","port":443,"targetPort":6443}],"clu [truncated 209 chars]
I0627 03:55:04.021674  116995 request.go:1212] Request Body: {"metadata":{"annotations":{"foo":null}}}
I0627 03:55:04.021707  116995 round_trippers.go:463] PATCH https://127.0.0.1:34887/api/v1/namespaces/default/services/kubernetes?fieldManager=kubectl-annotate
I0627 03:55:04.021710  116995 round_trippers.go:469] Request Headers:
I0627 03:55:04.021713  116995 round_trippers.go:473]     Accept: application/json
I0627 03:55:04.021715  116995 round_trippers.go:473]     Content-Type: application/merge-patch+json
I0627 03:55:04.021717  116995 round_trippers.go:473]     User-Agent: kubectl/v1.30.2 (linux/amd64) kubernetes/3968350
I0627 03:55:04.024471  116995 round_trippers.go:574] Response Status: 200 OK in 2 milliseconds
I0627 03:55:04.024492  116995 round_trippers.go:577] Response Headers:
I0627 03:55:04.024497  116995 round_trippers.go:580]     X-Kubernetes-Pf-Flowschema-Uid: ab36eba1-08ff-4a8a-bf6f-714d794ea8a3
I0627 03:55:04.024500  116995 round_trippers.go:580]     X-Kubernetes-Pf-Prioritylevel-Uid: 0626e4f8-723a-400f-a19d-528c77df29f4
I0627 03:55:04.024503  116995 round_trippers.go:580]     Content-Length: 1017
I0627 03:55:04.024506  116995 round_trippers.go:580]     Date: Thu, 27 Jun 2024 03:55:04 GMT
I0627 03:55:04.024508  116995 round_trippers.go:580]     Audit-Id: 0eab9a94-dd3d-4d3d-8f0f-643bcc2309b6
I0627 03:55:04.024510  116995 round_trippers.go:580]     Cache-Control: no-cache, private
I0627 03:55:04.024512  116995 round_trippers.go:580]     Content-Type: application/json
I0627 03:55:04.024526  116995 request.go:1212] Response Body: {"kind":"Service","apiVersion":"v1","metadata":{"name":"kubernetes","namespace":"default","uid":"692799b5-7532-4291-bcb6-d69dccf13def","resourceVersion":"536","creationTimestamp":"2024-06-27T03:52:42Z","labels":{"component":"apiserver","provider":"kubernetes"},"managedFields":[{"manager":"kube-apiserver","operation":"Update","apiVersion":"v1","time":"2024-06-27T03:52:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{".":{},"f:component":{},"f:provider":{}}},"f:spec":{"f:clusterIP":{},"f:internalTrafficPolicy":{},"f:ipFamilyPolicy":{},"f:ports":{".":{},"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:sessionAffinity":{},"f:type":{}}}}]},"spec":{"ports":[{"name":"https","protocol":"TCP","port":443,"targetPort":6443}],"clusterIP":"10.96.0.1","clusterIPs":["10.96.0.1"],"type":"ClusterIP","sessionAffinity":"None","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","internalTrafficPolicy":"Cluster"},"status":{"loadBalancer":{}}}
service/kubernetes annotated