Deployment meta annotations order broken

[leseb]

Describe the bug Since https://github.com/banzaicloud/k8s-objectmatcher/commit/4236369cea697476fc5d834d1e5b6e80f99a3e35, every timeSetLastAppliedAnnotation() is applied the .metadata.annotations of my Deployment has a different order.

See, the first iteration:

2020-03-23 13:50:13.544252 I | ceph-object-controller: deployment.Meta.Annotations is map[banzaicloud.com/last-applied:{"status":{},"metadata":{"name":"rook-ceph-rgw-my-store-b","namespace":"rook-ceph","labels":{"rgw":"my-store","rook_object_store":"my-store","rook-version":"v1.1.0-beta.0.1104.g803f3bced-dirty","ceph-version":"14.2.8-0","app":"rook-ceph-rgw","rook_cluster":"rook-ceph","ceph_daemon_id":"my-store"},"ownerReferences":[{"kind":"CephObjectStore","name":"my-store","uid":"9d7fe2b7-34fa-4d3a-9133-2ba94a9b9b44","controller":true,"blockOwnerDeletion":true,"apiVersion":"ceph.rook.io/v1"}]},"spec":{"selector":{"matchLabels":{"app":"rook-ceph-rgw","rook_cluster":"rook-ceph","ceph_daemon_id":"my-store","rgw":"my-store","rook_object_store":"my-store"}},"template":{"metadata":{"labels":{"rgw":"my-store","rook_object_store":"my-store","app":"rook-ceph-rgw","rook_cluster":"rook-ceph","ceph_daemon_id":"my-store"},"name":"rook-ceph-rgw-my-store-b"},"spec":{"volumes":[{"name":"rook-config-override","configMap":{"name":"rook-config-override","items":[{"path":"ceph.conf","mode":292,"key":"config"}]}},{"name":"rook-ceph-rgw-my-store-b-keyring","secret":{"secretName":"rook-ceph-rgw-my-store-b-keyring"}},{"hostPath":{"path":"/var/lib/rook/rook-ceph/log"},"name":"rook-ceph-log"},{"name":"rook-ceph-crash","hostPath":{"path":"/var/lib/rook/rook-ceph/crash"}},{"name":"ceph-daemon-data","emptyDir":{}},{"name":"rook-ceph-rgw-my-store-mime-types","configMap":{"name":"rook-ceph-rgw-my-store-mime-types"}}],"initContainers":[{"command":["chown"],"args":["--verbose","--recursive","ceph:ceph","/var/log/ceph","/var/lib/ceph/crash","/var/lib/ceph/rgw/ceph-my-store"],"resources":{},"volumeMounts":[{"name":"rook-config-override","readOnly":true,"mountPath":"/etc/ceph"},{"readOnly":true,"mountPath":"/etc/ceph/keyring-store/","name":"rook-ceph-rgw-my-store-b-keyring"},{"name":"rook-ceph-log","mountPath":"/var/log/ceph"},{"name":"rook-ceph-crash","mountPath":"/var/lib/ceph/crash"},{"name":"ceph-daemon-data","mountPath":"/var/lib/ceph/rgw/ceph-my-store"}],"name":"chown-container-data-dir","image":"ceph/ceph:v14.2.8"}],"containers":[{"image":"ceph/ceph:v14.2.8","command":["radosgw"],"env":[{"name":"CONTAINER_IMAGE","value":"ceph/ceph:v14.2.8"},{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"POD_MEMORY_LIMIT","valueFrom":{"resourceFieldRef":{"resource":"limits.memory","divisor":"0"}}},{"valueFrom":{"resourceFieldRef":{"resource":"requests.memory","divisor":"0"}},"name":"POD_MEMORY_REQUEST"},{"name":"POD_CPU_LIMIT","valueFrom":{"resourceFieldRef":{"resource":"limits.cpu","divisor":"1"}}},{"valueFrom":{"resourceFieldRef":{"divisor":"0","resource":"requests.cpu"}},"name":"POD_CPU_REQUEST"},{"name":"ROOK_CEPH_MON_HOST","valueFrom":{"secretKeyRef":{"name":"rook-ceph-config","key":"mon_host"}}},{"name":"ROOK_CEPH_MON_INITIAL_MEMBERS","valueFrom":{"secretKeyRef":{"name":"rook-ceph-config","key":"mon_initial_members"}}}],"livenessProbe":{"httpGet":{"path":"/swift/healthcheck","port":80},"initialDelaySeconds":10},"name":"rgw","args":["--fsid=2a769bde-4386-46ce-a23e-0b634a591a40","--keyring=/etc/ceph/keyring-store/keyring","--log-to-stderr=true","--err-to-stderr=true","--mon-cluster-log-to-stderr=true","--log-stderr-prefix=debug ","--default-log-to-file=false","--default-mon-cluster-log-to-file=false","--mon-host=$(ROOK_CEPH_MON_HOST)","--mon-initial-members=$(ROOK_CEPH_MON_INITIAL_MEMBERS)","--id=rgw.my.store.b","--setuser=ceph","--setgroup=ceph","--foreground","--rgw-frontends=beast port=80","--host=$(POD_NAME)","--rgw-mime-types-file=/etc/ceph/rgw/mime.types"],"resources":{},"volumeMounts":[{"mountPath":"/etc/ceph","name":"rook-config-override","readOnly":true},{"mountPath":"/etc/ceph/keyring-store/","name":"rook-ceph-rgw-my-store-b-keyring","readOnly":true},{"name":"rook-ceph-log","mountPath":"/var/log/ceph"},{"name":"rook-ceph-crash","mountPath":"/var/lib/ceph/crash"},{"name":"ceph-daemon-data","mountPath":"/var/lib/ceph/rgw/ceph-my-store"},{"readOnly":true,"mountPath":"/etc/ceph/rgw","name":"rook-ceph-rgw-my-store-mime-types"}]}],"restartPolicy":"Always","affinity":{"podAntiAffinity":{}},"tolerations":[{"tolerationSeconds":5,"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute"}]}},"strategy":{"type":"Recreate"},"replicas":1}}]

Second iteration:

2020-03-23 13:50:10.838079 I | ceph-object-controller: deployment.Meta.Annotations is map[banzaicloud.com/last-applied:{"metadata":{"labels":{"ceph-version":"14.2.8-0","app":"rook-ceph-rgw","rook_cluster":"rook-ceph","ceph_daemon_id":"my-store","rgw":"my-store","rook_object_store":"my-store","rook-version":"v1.1.0-beta.0.1104.g803f3bced-dirty"},"ownerReferences":[{"kind":"CephObjectStore","name":"my-store","uid":"9d7fe2b7-34fa-4d3a-9133-2ba94a9b9b44","controller":true,"blockOwnerDeletion":true,"apiVersion":"ceph.rook.io/v1"}],"name":"rook-ceph-rgw-my-store-a","namespace":"rook-ceph"},"spec":{"strategy":{"type":"Recreate"},"replicas":1,"selector":{"matchLabels":{"ceph_daemon_id":"my-store","rgw":"my-store","rook_object_store":"my-store","app":"rook-ceph-rgw","rook_cluster":"rook-ceph"}},"template":{"metadata":{"name":"rook-ceph-rgw-my-store-a","labels":{"app":"rook-ceph-rgw","rook_cluster":"rook-ceph","ceph_daemon_id":"my-store","rgw":"my-store","rook_object_store":"my-store"}},"spec":{"volumes":[{"name":"rook-config-override","configMap":{"items":[{"mode":292,"key":"config","path":"ceph.conf"}],"name":"rook-config-override"}},{"secret":{"secretName":"rook-ceph-rgw-my-store-a-keyring"},"name":"rook-ceph-rgw-my-store-a-keyring"},{"name":"rook-ceph-log","hostPath":{"path":"/var/lib/rook/rook-ceph/log"}},{"hostPath":{"path":"/var/lib/rook/rook-ceph/crash"},"name":"rook-ceph-crash"},{"emptyDir":{},"name":"ceph-daemon-data"},{"name":"rook-ceph-rgw-my-store-mime-types","configMap":{"name":"rook-ceph-rgw-my-store-mime-types"}}],"initContainers":[{"volumeMounts":[{"mountPath":"/etc/ceph","name":"rook-config-override","readOnly":true},{"readOnly":true,"mountPath":"/etc/ceph/keyring-store/","name":"rook-ceph-rgw-my-store-a-keyring"},{"name":"rook-ceph-log","mountPath":"/var/log/ceph"},{"name":"rook-ceph-crash","mountPath":"/var/lib/ceph/crash"},{"name":"ceph-daemon-data","mountPath":"/var/lib/ceph/rgw/ceph-my-store"}],"name":"chown-container-data-dir","image":"ceph/ceph:v14.2.8","command":["chown"],"args":["--verbose","--recursive","ceph:ceph","/var/log/ceph","/var/lib/ceph/crash","/var/lib/ceph/rgw/ceph-my-store"],"resources":{}}],"containers":[{"name":"rgw","image":"ceph/ceph:v14.2.8","command":["radosgw"],"env":[{"name":"CONTAINER_IMAGE","value":"ceph/ceph:v14.2.8"},{"name":"POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}},"name":"POD_NAMESPACE"},{"valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}},"name":"NODE_NAME"},{"name":"POD_MEMORY_LIMIT","valueFrom":{"resourceFieldRef":{"resource":"limits.memory","divisor":"0"}}},{"name":"POD_MEMORY_REQUEST","valueFrom":{"resourceFieldRef":{"resource":"requests.memory","divisor":"0"}}},{"name":"POD_CPU_LIMIT","valueFrom":{"resourceFieldRef":{"resource":"limits.cpu","divisor":"1"}}},{"name":"POD_CPU_REQUEST","valueFrom":{"resourceFieldRef":{"resource":"requests.cpu","divisor":"0"}}},{"valueFrom":{"secretKeyRef":{"name":"rook-ceph-config","key":"mon_host"}},"name":"ROOK_CEPH_MON_HOST"},{"name":"ROOK_CEPH_MON_INITIAL_MEMBERS","valueFrom":{"secretKeyRef":{"name":"rook-ceph-config","key":"mon_initial_members"}}}],"resources":{},"livenessProbe":{"initialDelaySeconds":10,"httpGet":{"path":"/swift/healthcheck","port":80}},"args":["--fsid=2a769bde-4386-46ce-a23e-0b634a591a40","--keyring=/etc/ceph/keyring-store/keyring","--log-to-stderr=true","--err-to-stderr=true","--mon-cluster-log-to-stderr=true","--log-stderr-prefix=debug ","--default-log-to-file=false","--default-mon-cluster-log-to-file=false","--mon-host=$(ROOK_CEPH_MON_HOST)","--mon-initial-members=$(ROOK_CEPH_MON_INITIAL_MEMBERS)","--id=rgw.my.store.a","--setuser=ceph","--setgroup=ceph","--foreground","--rgw-frontends=beast port=80","--host=$(POD_NAME)","--rgw-mime-types-file=/etc/ceph/rgw/mime.types"],"volumeMounts":[{"name":"rook-config-override","readOnly":true,"mountPath":"/etc/ceph"},{"readOnly":true,"mountPath":"/etc/ceph/keyring-store/","name":"rook-ceph-rgw-my-store-a-keyring"},{"name":"rook-ceph-log","mountPath":"/var/log/ceph"},{"mountPath":"/var/lib/ceph/crash","name":"rook-ceph-crash"},{"mountPath":"/var/lib/ceph/rgw/ceph-my-store","name":"ceph-daemon-data"},{"mountPath":"/etc/ceph/rgw","name":"rook-ceph-rgw-my-store-mime-types","readOnly":true}]}],"restartPolicy":"Always","affinity":{"podAntiAffinity":{}},"tolerations":[{"tolerationSeconds":5,"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute"}]}}},"status":{}}]

Basically .metadata.labels and .metadata.name are reversed.

Steps to reproduce the issue:

Deploy a Deployment resource with Labels.

Expected behavior

Annotations will get the same order all the time so that I don't get an endless reconcile loop.

Additional context

v1.1.0 worked well, the bug was introduced in v1.1.1 with https://github.com/banzaicloud/k8s-objectmatcher/commit/4236369cea697476fc5d834d1e5b6e80f99a3e35.

Thanks in advance for your help.

[eachirei]

Hi! I've stumbled upon the same issue and I think it's related to the use of json "github.com/json-iterator/go" instead of the default "encoding/json". The json-iterator package exposes a config with sorting enabled ConfigCompatibleWithStandardLibrary but the DeleteNullInJson and other methods uses the default config which doesn't include sorting. I've created a fork with the modifications (which work at first sight), but I don't know if there are other implications to these modifications: https://github.com/eachirei/k8s-objectmatcher/releases/tag/v1.3.3-alpha-2.

[pepov]

@eachirei sounds reasonable. Would you mind opening a PR?

[eachirei]

Just did it ^