search

karmada-io / karmada

Open, Multi-Cloud, Multi-Cluster Kubernetes Orchestration
https://karmada.io
Apache License 2.0
4.45k stars 882 forks source link

MultiClusterService is not effact for statefulset #4604

Open 13567436138 opened 8 months ago

13567436138 commented 8 months ago

sts-nginx.yaml

kubectl apply -f sts-nginx.yaml --kubeconfig ~/kube-karmada

apiVersion: v1
kind: Service
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  ports:
  - port: 80
    name: web
    targetPort: 80
  selector:
    app: nginx
---
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-svc
spec:
  propagateDeps: true
  resourceSelectors:
    - apiVersion: v1
      kind: Service
      name: nginx
  placement:
    clusterAffinity:
      clusterNames:
        - cluster1
        - cluster2 
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-headless
  labels:
    app: nginx
spec:
  ports:
  - port: 80
    name: web
  clusterIP: None
  selector:
    app: nginx
---
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-svc-headless
spec:
  propagateDeps: true
  resourceSelectors:
    - apiVersion: v1
      kind: Service
      name: nginx-headless
  placement:
    clusterAffinity:
      clusterNames:
        - cluster1
        - cluster2 
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: web
spec:
  ordinals:
    start: 0
  selector:
    matchLabels:
      app: nginx
  serviceName: "nginx-headless"
  replicas: 3
  template:
    metadata:
      labels:
        app: nginx
    spec:
      terminationGracePeriodSeconds: 10
      containers:
      - name: nginx
        image: nginx
        ports:
        - containerPort: 80
          name: web
        volumeMounts:
        - name: www
          mountPath: /usr/share/nginx/html
  volumeClaimTemplates:
  - metadata:
      name: www
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "ceph-rbd"
      resources:
        requests:
          storage: 10Mi
---
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-deploy
spec:
  propagateDeps: true
  failover:
    application:
      decisionConditions:
        tolerationSeconds: 300
      gracePeriodSeconds: 10
      purgeMode: Graciously
  resourceSelectors:
    - apiVersion: apps/v1
      kind: StatefulSet
      name: web
  placement:
    clusterAffinity:
      clusterNames:
        - cluster1
        - cluster2
---
apiVersion: policy.karmada.io/v1alpha1
kind: OverridePolicy
metadata:
  name: nginx-member2
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: StatefulSet
      name: web
  targetCluster:
    clusterNames:
      - cluster2
  overriders:
    plaintext:
      - path: "/spec/ordinals/start"
        operator: replace
        value: 3

cluster1

root@karmada-02-a:~# kubectl get pod -w
NAME    READY   STATUS    RESTARTS   AGE
web-0   1/1     Running   0          4m18s
web-1   1/1     Running   0          3m45s
web-2   1/1     Running   0          3m3s

cluster2

root@karmada-03-a:~# kubectl get pod -w
NAME    READY   STATUS    RESTARTS   AGE
web-3   1/1     Running   0          5m20s
web-4   1/1     Running   0          4m53s
web-5   1/1     Running   0          4m40s

multiClusterService-nginx.yaml

kubectl apply -f multiClusterService-nginx.yaml --kubeconfig ~/kube-karmada

---
apiVersion: networking.karmada.io/v1alpha1
kind: MultiClusterService
metadata:
   name: nginx
spec:
   types:
   - CrossCluster
   serviceProvisionClusters:
   - cluster1
   - cluster2
   serviceConsumptionClusters:
   - cluster1
   - cluster2

cluster1

kubectl exec web-0 -- sh -c 'echo web-0 > /usr/share/nginx/html/index.html'
kubectl exec web-1 -- sh -c 'echo web-1 > /usr/share/nginx/html/index.html'
kubectl exec web-2 -- sh -c 'echo web-2 > /usr/share/nginx/html/index.html'

cluster2

kubectl exec web-3 -- sh -c 'echo web-3 > /usr/share/nginx/html/index.html'
kubectl exec web-4 -- sh -c 'echo web-4 > /usr/share/nginx/html/index.html'
kubectl exec web-5 -- sh -c 'echo web-5 > /usr/share/nginx/html/index.html'
root@karmada-02-a:~# curl 10.69.240.105
web-2
root@karmada-02-a:~# curl 10.69.240.105
web-1
root@karmada-02-a:~# curl 10.69.240.105
web-0
root@karmada-02-a:~# curl 10.69.240.105
web-2
root@karmada-02-a:~# curl 10.69.240.105
web-1
root@karmada-02-a:~# curl 10.69.240.105
web-0
root@karmada-02-a:~# curl 10.69.240.105
web-2
root@karmada-02-a:~# kubectl get endpointslices.discovery.k8s.io 
NAME                   ADDRESSTYPE   PORTS   ENDPOINTS                         AGE
kubernetes             IPv4          6443    192.168.229.182                   23h
nginx-h2m6q            IPv4          80      10.2.1.192,10.2.0.16,10.2.2.112   24m
nginx-headless-q697g   IPv4          80      10.2.1.192,10.2.0.16,10.2.2.112   24m

the endpointslice is not generated for annothor cluster What happened:

What you expected to happen:

How to reproduce it (as minimally and precisely as possible):

Anything else we need to know?:

Environment:

jwcesign commented 8 months ago

Hi, there are a few updates recently(bug fix related), can you try with the latest version(master branch).

I try the following yaml, and it works fine:

apiVersion: v1
kind: Service
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  ports:
  - port: 80
    name: web
    targetPort: 80
  selector:
    app: nginx
---
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-svc
spec:
  propagateDeps: true
  resourceSelectors:
    - apiVersion: v1
      kind: Service
      name: nginx
  placement:
    clusterAffinity:
      clusterNames:
        - member1
        - member2
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-headless
  labels:
    app: nginx
spec:
  ports:
  - port: 80
    name: web
  clusterIP: None
  selector:
    app: nginx
---
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-svc-headless
spec:
  propagateDeps: true
  resourceSelectors:
    - apiVersion: v1
      kind: Service
      name: nginx-headless
  placement:
    clusterAffinity:
      clusterNames:
        - member1
        - member2
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: web
spec:
  ordinals:
    start: 0
  selector:
    matchLabels:
      app: nginx
  serviceName: "nginx-headless"
  replicas: 3
  template:
    metadata:
      labels:
        app: nginx
    spec:
      terminationGracePeriodSeconds: 10
      containers:
      - name: nginx
        image: nginx
        ports:
        - containerPort: 80
          name: web
---
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-deploy
spec:
  propagateDeps: true
  failover:
    application:
      decisionConditions:
        tolerationSeconds: 300
      gracePeriodSeconds: 10
      purgeMode: Graciously
  resourceSelectors:
    - apiVersion: apps/v1
      kind: StatefulSet
      name: web
  placement:
    clusterAffinity:
      clusterNames:
        - member1
        - member2
---
#apiVersion: policy.karmada.io/v1alpha1
#kind: OverridePolicy
#metadata:
#  name: nginx-member2
#spec:
#  resourceSelectors:
#    - apiVersion: apps/v1
#      kind: StatefulSet
#      name: web
#  targetCluster:
#    clusterNames:
#      - member2
#  overriders:
#    plaintext:
#      - path: "/spec/ordinals/start"
#        operator: replace
#        value: 3
---
apiVersion: networking.karmada.io/v1alpha1
kind: MultiClusterService
metadata:
   name: nginx
spec:
   types:
   - CrossCluster
   providerClusters:
   - name: member1
   - name: member2
   consumerClusters:
   - name: member1
   - name: member2

The endpointslice:

root@karmada [02:31:26 PM] [~/workspace/demo/mcs/mcs-serviceexport]
-> # karmadactl get endpointslice
NAME                   CLUSTER   ADDRESSTYPE   PORTS   ENDPOINTS                          AGE     ADOPTION
kubernetes             member2   IPv4          6443    172.18.0.4                         4d22h   N
member1-nginx-h54rm    member2   IPv4          80      10.10.0.92,10.10.0.93,10.10.0.94   7m45s   Y
nginx-headless-hpddj   member2   IPv4          80      10.12.0.38,10.12.0.39,10.12.0.40   7m45s   Y
nginx-hgbfs            member2   IPv4          80      10.12.0.38,10.12.0.39,10.12.0.40   7m45s   Y
kubernetes             member1   IPv4          6443    172.18.0.5                         4d22h   N
member2-nginx-hgbfs    member1   IPv4          80      10.12.0.38,10.12.0.39,10.12.0.40   7m45s   Y
nginx-h54rm            member1   IPv4          80      10.10.0.92,10.10.0.93,10.10.0.94   7m45s   Y
nginx-headless-2xx96   member1   IPv4          80      10.10.0.92,10.10.0.93,10.10.0.94   7m45s   Y
kubernetes             member3   IPv4          6443    172.18.0.3                         4d22h   N
kubernetes             member4   IPv4          6443    172.18.0.7                         3d19h   N
13567436138 commented 8 months ago

I update to latest version,the same no generated endpointslice

root@karmada-01-b:~/karmada-teaching/sts# karmadactl get endpointslice --kubeconfig ~/kube-karmada 
NAME                   CLUSTER    ADDRESSTYPE   PORTS   ENDPOINTS                         AGE   ADOPTION
kubernetes             cluster1   IPv4          6443    192.168.229.182                   47h   N
nginx-headless-k9fzk   cluster1   IPv4          80      10.2.1.141,10.2.0.131,10.2.3.54   12m   Y
nginx-wh5pl            cluster1   IPv4          80      10.2.1.141,10.2.0.131,10.2.3.54   12m   Y
kubernetes             cluster2   IPv4          6443    192.168.229.185                   56d   N
nginx-5dnr4            cluster2   IPv4          80      10.2.2.118,10.2.1.112,10.2.0.99   12m   Y
nginx-headless-lhcgm   cluster2   IPv4          80      10.2.2.118,10.2.1.112,10.2.0.99   12m   Y
kubernetes             cluster3   IPv4          6443    192.168.229.202                   13d   N
jwcesign commented 8 months ago

Can you show me the result of the following commands, with karmada-apiserver context? kubectl get mcs -oyaml

kubectl get work -A

13567436138 commented 8 months ago

你用的是什么网络

13567436138 commented 8 months ago 
root@karmada-01-b:~/karmada-teaching/sts# kubectl get mcs -oyaml --kubeconfig ~/kube-karmada 
apiVersion: v1
items:
- apiVersion: networking.karmada.io/v1alpha1
  kind: MultiClusterService
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"networking.karmada.io/v1alpha1","kind":"MultiClusterService","metadata":{"annotations":{},"name":"nginx","namespace":"default"},"spec":{"serviceConsumptionClusters":["cluster1","cluster2","cluster3"],"serviceProvisionClusters":["cluster1","cluster2","cluster3"],"types":["CrossCluster"]}}
    creationTimestamp: "2024-02-03T05:40:00Z"
    generation: 1
    labels:
      multiclusterservice.karmada.io/permanent-id: 6e9ac7c4-5477-46a6-b075-22b46e5aee70
    name: nginx
    namespace: default
    resourceVersion: "8715"
    uid: 6d78afa2-755f-45bb-aa1a-fb74547f923e
  spec:
    range: {}
    serviceConsumptionClusters:
    - cluster1
    - cluster2
    - cluster3
    serviceProvisionClusters:
    - cluster1
    - cluster2
    - cluster3
    types:
    - CrossCluster
kind: List
metadata:
  resourceVersion: ""
root@karmada-01-b:~/karmada-teaching/sts# kubectl get work -A --kubeconfig ~/kube-karmada 
NAMESPACE             NAME                              WORKLOAD-KIND        APPLIED   AGE
karmada-es-cluster1   karmada-impersonator-7cbb6bd5c9   ClusterRole          True      23h
karmada-es-cluster1   karmada-impersonator-84f8c8f8c6   ClusterRoleBinding   True      23h
karmada-es-cluster1   nginx-64b7849b5                   Service              True      23m
karmada-es-cluster1   nginx-headless-5ffd8b4dbc         Service              True      23m
karmada-es-cluster2   karmada-impersonator-7cbb6bd5c9   ClusterRole          True      23h
karmada-es-cluster2   karmada-impersonator-84f8c8f8c6   ClusterRoleBinding   True      23h
karmada-es-cluster2   nginx-64b7849b5                   Service              True      23m
karmada-es-cluster2   nginx-headless-5ffd8b4dbc         Service              True      23m
karmada-es-cluster2   web-7754dc7556                    StatefulSet          True      23m
karmada-es-cluster3   karmada-impersonator-7cbb6bd5c9   ClusterRole          True      23h
karmada-es-cluster3   karmada-impersonator-84f8c8f8c6   ClusterRoleBinding   True      23h
karmada-es-cluster3   web-7754dc7556                    StatefulSet          True      7m34s

multiClusterService-nginx.yaml

kubectl apply -f multiClusterService-nginx.yaml --kubeconfig ~/kube-karmada

---
apiVersion: networking.karmada.io/v1alpha1
kind: MultiClusterService
metadata:
   name: nginx
spec:
   types:
   - CrossCluster
   serviceProvisionClusters:
   - cluster1
   - cluster2
   - cluster3
   serviceConsumptionClusters:
   - cluster1
   - cluster2
   - cluster3
jwcesign commented 8 months ago

你用的是什么网络

I tested with kind, so the container network is kind-net(https://github.com/aojea/kindnet)

Could you assist by executing the following command?

kubectl explain mcs.spec --kubeconfig ~/kube-karmada

Also, are you using the latest version?

13567436138 commented 8 months ago

最新版的，cilium 用了cilium cluster mesh

13567436138 commented 8 months ago 
root@karmada-01-b:~/karmada-teaching/disater-recovery# kubectl explain mcs.spec --kubeconfig ~/kube-karmada 
E0204 20:57:03.029876  431679 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0204 20:57:03.374013  431679 memcache.go:287] couldn't get resource list for cluster.karmada.io/v1alpha1: the server is currently unable to handle the request
E0204 20:57:03.412788  431679 memcache.go:287] couldn't get resource list for custom.metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0204 20:57:03.418026  431679 memcache.go:287] couldn't get resource list for custom.metrics.k8s.io/v1beta2: the server is currently unable to handle the request
E0204 20:57:03.762126  431679 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0204 20:57:03.855161  431679 memcache.go:121] couldn't get resource list for cluster.karmada.io/v1alpha1: the server is currently unable to handle the request
E0204 20:57:04.016940  431679 memcache.go:121] couldn't get resource list for custom.metrics.k8s.io/v1beta2: the server is currently unable to handle the request
E0204 20:57:04.148608  431679 memcache.go:121] couldn't get resource list for custom.metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0204 20:57:04.168604  431679 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0204 20:57:04.175347  431679 memcache.go:121] couldn't get resource list for cluster.karmada.io/v1alpha1: the server is currently unable to handle the request
E0204 20:57:04.237105  431679 memcache.go:121] couldn't get resource list for custom.metrics.k8s.io/v1beta2: the server is currently unable to handle the request
E0204 20:57:04.391072  431679 memcache.go:121] couldn't get resource list for custom.metrics.k8s.io/v1beta1: the server is currently unable to handle the request
GROUP:      networking.karmada.io
KIND:       MultiClusterService
VERSION:    v1alpha1

FIELD: spec <Object>

DESCRIPTION:
    Spec is the desired state of the MultiClusterService.

FIELDS:
  consumerClusters      <[]Object>
    ConsumerClusters specifies the clusters where the service will be exposed,
    for clients. If leave it empty, the service will be exposed to all clusters.

  ports <[]Object>
    Ports is the list of ports that are exposed by this MultiClusterService. No
    specified port will be filtered out during the service exposure and
    discovery process. All ports in the referencing service will be exposed by
    default.

  providerClusters      <[]Object>
    ProviderClusters specifies the clusters which will provide the service
    backend. If leave it empty, we will collect the backend endpoints from all
    clusters and sync them to the ConsumerClusters.

  range <Object>
    Range specifies the ranges where the referencing service should be exposed.
    Only valid and optional in case of Types contains CrossCluster. If not set
    and Types contains CrossCluster, all clusters will be selected, that means
    the referencing service will be exposed across all registered clusters.
    Deprecated: in favor of ProviderClusters/ConsumerClusters.

  serviceConsumptionClusters    <[]string>
    ServiceConsumptionClusters specifies the clusters where the service will be
    exposed, for clients. If leave it empty, the service will be exposed to all
    clusters. Deprecated: in favor of ProviderClusters/ConsumerClusters.

  serviceProvisionClusters      <[]string>
    ServiceProvisionClusters specifies the clusters which will provision the
    service backend. If leave it empty, we will collect the backend endpoints
    from all clusters and sync them to the ServiceConsumptionClusters.
    Deprecated: in favor of ProviderClusters/ConsumerClusters.

  types <[]string> -required-
    Types specifies how to expose the service referencing by this
    MultiClusterService.
13567436138 commented 8 months ago

我的配置有点问题，我再试下

13567436138 commented 8 months ago 


root@karmada-01-b:~/karmada-teaching/sts# kubectl get mcs --kubeconfig ~/kube-karmada nginx -o yaml
apiVersion: networking.karmada.io/v1alpha1
kind: MultiClusterService
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"networking.karmada.io/v1alpha1","kind":"MultiClusterService","metadata":{"annotations":{},"name":"nginx","namespace":"default"},"spec":{"consumerClusters":[{"name":"cluster1"},{"name":"cluster2"},{"name":"cluster3"}],"providerClusters":[{"name":"cluster1"},{"name":"cluster2"},{"name":"cluster3"}],"types":["CrossCluster"]}}
  creationTimestamp: "2024-02-04T13:05:13Z"
  generation: 1
  labels:
    multiclusterservice.karmada.io/permanent-id: 9866b439-993c-4793-b837-8f3216daf558
  name: nginx
  namespace: default
  resourceVersion: "110597"
  uid: 82a62ddc-3156-48cd-b220-12c658e6eeb6
spec:
  consumerClusters:
  - name: cluster1
  - name: cluster2
  - name: cluster3
  providerClusters:
  - name: cluster1
  - name: cluster2
  - name: cluster3
  range: {}
  types:
  - CrossCluster```

改了还是不行
jwcesign commented 8 months ago

Can you show me the component log by running(xxx is determined by the specific environment):

kubectl logs -nkarmada-system karmada-controller-manager-xxx | grep "mcs_controller.go"

also, show me the output of command:

kubectl get work -A --kubeconfig ~/kube-karmada
13567436138 commented 8 months ago

明天发你

13567436138 commented 8 months ago 
root@karmada-01-b:~/karmada-teaching/sts# kubectl logs -n karmada-system karmada-controller-manager-7b587bcbfb-xkpsd| grep "mcs_controller.go"
root@karmada-01-b:~/karmada-teaching/sts# 
root@karmada-01-b:~/karmada-teaching/sts# kubectl get work -A --kubeconfig ~/kube-karmada 
NAMESPACE            NAME                              WORKLOAD-KIND        APPLIED   AGE
karmada-es-member1   karmada-impersonator-7cbb6bd5c9   ClusterRole          True      8m12s
karmada-es-member1   karmada-impersonator-84f8c8f8c6   ClusterRoleBinding   True      8m11s
karmada-es-member1   nginx-64b7849b5                   Service              True      3m17s
karmada-es-member1   nginx-headless-5ffd8b4dbc         Service              True      3m16s
karmada-es-member1   web-7754dc7556                    StatefulSet          True      3m16s
karmada-es-member2   karmada-impersonator-7cbb6bd5c9   ClusterRole          True      7m52s
karmada-es-member2   karmada-impersonator-84f8c8f8c6   ClusterRoleBinding   True      7m52s
karmada-es-member2   nginx-64b7849b5                   Service              True      3m16s
karmada-es-member2   nginx-headless-5ffd8b4dbc         Service              True      3m16s
karmada-es-member2   web-7754dc7556                    StatefulSet          True      3m15s
13567436138 commented 8 months ago

@jwcesign