[Bug]: kafka.zookeeper.ZooKeeperClientTimeoutException: Timed out waiting for connection while in state: CONNECTING

[hf185]

Is there an existing issue for this?

• [X] I have searched the existing issues

Environment

- Milvus version: v2.2.11
- Deployment mode(standalone or cluster): cluster
- MQ type(rocksmq, pulsar or kafka):kafka
- SDK version(e.g. pymilvus v2.0.0rc2):
- OS(Ubuntu or CentOS): CentOS
- CPU/Memory: (64/256)*4
- GPU: 
- Others:

Current Behavior

Using k8s to deploy the milvus cluster fails, and the kafka log shows that the connection to zk fails

Expected Behavior

No response

Steps To Reproduce

This is my milvus_manifest.yaml

---
# Source: milvus/charts/kafka/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: my-milvus-kafka
  namespace: "default"
  labels:
    app.kubernetes.io/name: kafka
    helm.sh/chart: kafka-15.5.1
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: kafka
  annotations:
automountServiceAccountToken: true
---
# Source: milvus/charts/minio/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: "my-milvus-minio"
  namespace: "default"
  labels:
    app: minio
    chart: minio-8.0.15
    release: "my-milvus"
---
# Source: milvus/charts/minio/templates/secrets.yaml
apiVersion: v1
kind: Secret
metadata:
  name: my-milvus-minio
  labels:
    app: minio
    chart: minio-8.0.15
    release: my-milvus
    heritage: Helm
type: Opaque
data:
  accesskey: "bWluaW9hZG1pbg=="
  secretkey: "bWluaW9hZG1pbg=="
---
# Source: milvus/charts/kafka/charts/zookeeper/templates/scripts-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: my-milvus-zookeeper-scripts
  namespace: default
  labels:
    app.kubernetes.io/name: zookeeper
    helm.sh/chart: zookeeper-8.1.2
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: zookeeper
data:
  init-certs.sh: |-
    #!/bin/bash
  setup.sh: |-
    #!/bin/bash

    # Execute entrypoint as usual after obtaining ZOO_SERVER_ID
    # check ZOO_SERVER_ID in persistent volume via myid
    # if not present, set based on POD hostname
    if [[ -f "/bitnami/zookeeper/data/myid" ]]; then
        export ZOO_SERVER_ID="$(cat /bitnami/zookeeper/data/myid)"
    else
        HOSTNAME="$(hostname -s)"
        if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then
            ORD=${BASH_REMATCH[2]}
            export ZOO_SERVER_ID="$((ORD + 1 ))"
        else
            echo "Failed to get index from hostname $HOST"
            exit 1
        fi
    fi
    exec /entrypoint.sh /run.sh
---
# Source: milvus/charts/kafka/templates/scripts-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: my-milvus-kafka-scripts
  namespace: "default"
  labels:
    app.kubernetes.io/name: kafka
    helm.sh/chart: kafka-15.5.1
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
data:
  setup.sh: |-
    #!/bin/bash

    ID="${MY_POD_NAME#"my-milvus-kafka-"}"
    if [[ -f "/bitnami/kafka/data/meta.properties" ]]; then
        export KAFKA_CFG_BROKER_ID="$(grep "broker.id" /bitnami/kafka/data/meta.properties | awk -F '=' '{print $2}')"
    else
        export KAFKA_CFG_BROKER_ID="$((ID + 0))"
    fi

    exec /entrypoint.sh /run.sh
---
# Source: milvus/charts/minio/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: my-milvus-minio
  labels:
    app: minio
    chart: minio-8.0.15
    release: my-milvus
    heritage: Helm
data:
  initialize: |-
    #!/bin/sh
    set -e ; # Have script exit in the event of a failed command.
    MC_CONFIG_DIR="/etc/minio/mc/"
    MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"

    # connectToMinio
    # Use a check-sleep-check loop to wait for Minio service to be available
    connectToMinio() {
      SCHEME=$1
      ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
      set -e ; # fail if we can't read the keys.
      ACCESS=$(cat /config/accesskey) ; SECRET=$(cat /config/secretkey) ;
      set +e ; # The connections to minio are allowed to fail.
      echo "Connecting to Minio server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
      MC_COMMAND="${MC} config host add myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
      $MC_COMMAND ;
      STATUS=$? ;
      until [ $STATUS = 0 ]
      do
        ATTEMPTS=`expr $ATTEMPTS + 1` ;
        echo \"Failed attempts: $ATTEMPTS\" ;
        if [ $ATTEMPTS -gt $LIMIT ]; then
          exit 1 ;
        fi ;
        sleep 2 ; # 1 second intervals between attempts
        $MC_COMMAND ;
        STATUS=$? ;
      done ;
      set -e ; # reset `e` as active
      return 0
    }

    # checkBucketExists ($bucket)
    # Check if the bucket exists, by using the exit code of `mc ls`
    checkBucketExists() {
      BUCKET=$1
      CMD=$(${MC} ls myminio/$BUCKET > /dev/null 2>&1)
      return $?
    }

    # createBucket ($bucket, $policy, $purge)
    # Ensure bucket exists, purging if asked to
    createBucket() {
      BUCKET=$1
      POLICY=$2
      PURGE=$3
      VERSIONING=$4

      # Purge the bucket, if set & exists
      # Since PURGE is user input, check explicitly for `true`
      if [ $PURGE = true ]; then
        if checkBucketExists $BUCKET ; then
          echo "Purging bucket '$BUCKET'."
          set +e ; # don't exit if this fails
          ${MC} rm -r --force myminio/$BUCKET
          set -e ; # reset `e` as active
        else
          echo "Bucket '$BUCKET' does not exist, skipping purge."
        fi
      fi

      # Create the bucket if it does not exist
      if ! checkBucketExists $BUCKET ; then
        echo "Creating bucket '$BUCKET'"
        ${MC} mb myminio/$BUCKET
      else
        echo "Bucket '$BUCKET' already exists."
      fi

      # set versioning for bucket
      if [ ! -z $VERSIONING ] ; then
        if [ $VERSIONING = true ] ; then
            echo "Enabling versioning for '$BUCKET'"
            ${MC} version enable myminio/$BUCKET
        elif [ $VERSIONING = false ] ; then
            echo "Suspending versioning for '$BUCKET'"
            ${MC} version suspend myminio/$BUCKET
        fi
      else
          echo "Bucket '$BUCKET' versioning unchanged."
      fi

      # At this point, the bucket should exist, skip checking for existence
      # Set policy on the bucket
      echo "Setting policy of bucket '$BUCKET' to '$POLICY'."
      ${MC} policy set $POLICY myminio/$BUCKET
    }

    # Try connecting to Minio instance
    scheme=http
    connectToMinio $scheme
---
# Source: milvus/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: my-milvus-milvus
data:
  default.yaml: |+
    # Copyright (C) 2019-2021 Zilliz. All rights reserved.
    #
    # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
    # with the License. You may obtain a copy of the License at
    #
    # http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software distributed under the License
    # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
    # or implied. See the License for the specific language governing permissions and limitations under the License.

    etcd:
      endpoints:
        - my-milvus-etcd:2379

    metastore:
      type: etcd

    minio:
      address: my-milvus-minio
      port: 9000
      accessKeyID: minioadmin
      secretAccessKey: minioadmin
      useSSL: false
      bucketName: milvus-bucket
      rootPath: file
      useIAM: false
      iamEndpoint: 

    messageQueue: kafka

    kafka:
      brokerList: my-milvus-kafka:9092

    rootCoord:
      address: my-milvus-milvus-rootcoord
      port: 53100
      enableActiveStandby: false  # Enable active-standby

    proxy:
      port: 19530
      internalPort: 19529

    queryCoord:
      address: my-milvus-milvus-querycoord
      port: 19531

      enableActiveStandby: false  # Enable active-standby

    queryNode:
      port: 21123
      enableDisk: true # Enable querynode load disk index, and search on disk index

    indexCoord:
      address: my-milvus-milvus-indexcoord
      port: 31000
      enableActiveStandby: false  # Enable active-standby

    indexNode:
      port: 21121
      enableDisk: true # Enable index node build disk vector index

    dataCoord:
      address: my-milvus-milvus-datacoord
      port: 13333
      enableActiveStandby: false  # Enable active-standby

    dataNode:
      port: 21124

    log:
      level: info
      file:
        rootPath: ""
        maxSize: 300
        maxAge: 10
        maxBackups: 20
      format: text
  user.yaml: |-
    #    For example enable rest http for milvus proxy
    #    proxy:
    #      http:
    #        enabled: true
---
# Source: milvus/charts/etcd/templates/svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-etcd-headless
  namespace: default
  labels:
    app.kubernetes.io/name: etcd
    helm.sh/chart: etcd-6.3.3
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
  type: ClusterIP
  clusterIP: None
  publishNotReadyAddresses: true
  ports:
    - name: "client"
      port: 2379
      targetPort: client
    - name: "peer"
      port: 2380
      targetPort: peer
  selector:
    app.kubernetes.io/name: etcd
    app.kubernetes.io/instance: my-milvus
---
# Source: milvus/charts/etcd/templates/svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-etcd
  namespace: default
  labels:
    app.kubernetes.io/name: etcd
    helm.sh/chart: etcd-6.3.3
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
  annotations:
spec:
  type: ClusterIP
  ports:
    - name: "client"
      port: 2379
      targetPort: client
      nodePort: null
    - name: "peer"
      port: 2380
      targetPort: peer
      nodePort: null
  selector:
    app.kubernetes.io/name: etcd
    app.kubernetes.io/instance: my-milvus
---
# Source: milvus/charts/kafka/charts/zookeeper/templates/svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-zookeeper-headless
  namespace: default
  labels:
    app.kubernetes.io/name: zookeeper
    helm.sh/chart: zookeeper-8.1.2
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: zookeeper
spec:
  type: ClusterIP
  clusterIP: None
  publishNotReadyAddresses: true
  ports:
    - name: tcp-client
      port: 2181
      targetPort: client
    - name: tcp-follower
      port: 2888
      targetPort: follower
    - name: tcp-election
      port: 3888
      targetPort: election
  selector:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/component: zookeeper
---
# Source: milvus/charts/kafka/charts/zookeeper/templates/svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-zookeeper
  namespace: default
  labels:
    app.kubernetes.io/name: zookeeper
    helm.sh/chart: zookeeper-8.1.2
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: zookeeper
spec:
  type: ClusterIP
  sessionAffinity: None
  ports:
    - name: tcp-client
      port: 2181
      targetPort: client
      nodePort: null
    - name: tcp-follower
      port: 2888
      targetPort: follower
    - name: tcp-election
      port: 3888
      targetPort: election
  selector:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/component: zookeeper
---
# Source: milvus/charts/kafka/templates/svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-kafka-headless
  namespace: "default"
  labels:
    app.kubernetes.io/name: kafka
    helm.sh/chart: kafka-15.5.1
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: kafka
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: tcp-client
      port: 9092
      protocol: TCP
      targetPort: kafka-client
    - name: tcp-internal
      port: 9093
      protocol: TCP
      targetPort: kafka-internal
  selector:
    app.kubernetes.io/name: kafka
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/component: kafka
---
# Source: milvus/charts/kafka/templates/svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-kafka
  namespace: "default"
  labels:
    app.kubernetes.io/name: kafka
    helm.sh/chart: kafka-15.5.1
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: kafka
spec:
  type: ClusterIP
  sessionAffinity: None
  ports:
    - name: tcp-client
      port: 9092
      protocol: TCP
      targetPort: kafka-client
      nodePort: null
  selector:
    app.kubernetes.io/name: kafka
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/component: kafka
---
# Source: milvus/charts/minio/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-minio
  labels:
    app: minio
    chart: minio-8.0.15
    release: my-milvus
    heritage: Helm
spec:
  type: ClusterIP
  ports:
    - name: http
      port: 9000
      protocol: TCP
      targetPort: 9000
  selector:
    app: minio
    release: my-milvus
---
# Source: milvus/charts/minio/templates/statefulset.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-minio-svc
  labels:
    app: minio
    chart: minio-8.0.15
    release: "my-milvus"
    heritage: "Helm"
spec:
  publishNotReadyAddresses: true
  clusterIP: None
  ports:
    - name: http
      port: 9000
      protocol: TCP
  selector:
    app: minio
    release: my-milvus
---
# Source: milvus/templates/attu-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus-attu
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "attu"
spec:
  type: ClusterIP
  ports:
    - name: attu
      protocol: TCP
      port: 3000
      targetPort: 3000
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "attu"
---
# Source: milvus/templates/datacoord-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus-datacoord
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "datacoord"
spec:
  type: ClusterIP
  ports:
    - name: datacoord
      port: 13333
      protocol: TCP
      targetPort: datacoord
    - name: metrics
      protocol: TCP
      port: 9091
      targetPort: metrics
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "datacoord"
---
# Source: milvus/templates/datanode-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus-datanode
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "datanode"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: metrics
      protocol: TCP
      port: 9091
      targetPort: metrics
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "datanode"
---
# Source: milvus/templates/indexcoord-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus-indexcoord
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "indexcoord"
spec:
  type: ClusterIP
  ports:
    - name: indexcoord
      port: 31000
      protocol: TCP
      targetPort: indexcoord
    - name: metrics
      protocol: TCP
      port: 9091
      targetPort: metrics
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "indexcoord"
---
# Source: milvus/templates/indexnode-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus-indexnode
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "indexnode"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: metrics
      protocol: TCP
      port: 9091
      targetPort: metrics
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "indexnode"
---
# Source: milvus/templates/querycoord-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus-querycoord
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "querycoord"
spec:
  type: ClusterIP
  ports:
    - name: querycoord
      port: 19531
      protocol: TCP
      targetPort: querycoord
    - name: metrics
      protocol: TCP
      port: 9091
      targetPort: metrics
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "querycoord"
---
# Source: milvus/templates/querynode-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus-querynode
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "querynode"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: metrics
      protocol: TCP
      port: 9091
      targetPort: metrics
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "querynode"
---
# Source: milvus/templates/rootcoord-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus-rootcoord
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "rootcoord"
spec:
  type: ClusterIP
  ports:
    - name: rootcoord
      port: 53100
      protocol: TCP
      targetPort: rootcoord
    - name: metrics
      protocol: TCP
      port: 9091
      targetPort: metrics
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "rootcoord"
---
# Source: milvus/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: my-milvus-milvus
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "proxy"
spec:
  type: ClusterIP
  ports:
    - name: milvus
      port: 19530
      protocol: TCP
      targetPort: milvus
    - name: metrics
      protocol: TCP
      port: 9091
      targetPort: metrics
  selector:
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    component: "proxy"
---
# Source: milvus/templates/attu-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-attu
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "attu"
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "attu"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus
        component: "attu"
    spec:
      containers:
      - name: attu
        image: zilliz/attu:v2.2.4
        imagePullPolicy: IfNotPresent
        ports:
        - name: attu
          containerPort: 3000
          protocol: TCP
        env:
        - name: MILVUS_URL
          value: http://my-milvus-milvus:19530
        resources:
          {}
---
# Source: milvus/templates/datacoord-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-datacoord
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "datacoord"

  annotations:

spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "datacoord"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus
        component: "datacoord"

      annotations:
        pyroscope.io/scrape: "true"
        pyroscope.io/application-name: my-milvus-milvus-datacoord
        pyroscope.io/port: "9091"
        checksum/config: b16c4c9fe32787486b1ac5c88467f7e3ddb64d9e0f97e9f9b827434867dc474c

    spec:
      serviceAccountName: default
      initContainers:
      - name: config
        command:
        - /cp
        - /run-helm.sh,/merge
        - /milvus/tools/run-helm.sh,/milvus/tools/merge
        image: "milvusdb/milvus-config-tool:v0.1.1"
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /milvus/tools
          name: tools
      containers:
      - name: datacoord
        image: "milvusdb/milvus:v2.2.11"
        imagePullPolicy: IfNotPresent
        args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "datacoord" ]
        env:
        ports:
          - name: datacoord
            containerPort: 13333
            protocol: TCP
          - name: metrics
            containerPort: 9091
            protocol: TCP
        livenessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 30
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        resources:
          {}
        volumeMounts:
        - name: milvus-config
          mountPath: /milvus/configs/default.yaml
          subPath: default.yaml
          readOnly: true
        - name: milvus-config
          mountPath: /milvus/configs/user.yaml
          subPath: user.yaml
          readOnly: true
        - mountPath: /milvus/tools
          name: tools

      volumes:
      - name: milvus-config
        configMap:
          name: my-milvus-milvus
      - name: tools
        emptyDir: {}
---
# Source: milvus/templates/datanode-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-datanode
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "datanode"

  annotations:

spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "datanode"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus
        component: "datanode"

      annotations:
        checksum/config: b16c4c9fe32787486b1ac5c88467f7e3ddb64d9e0f97e9f9b827434867dc474c

    spec:
      serviceAccountName: default
      initContainers:
      - name: config
        command:
        - /cp
        - /run-helm.sh,/merge
        - /milvus/tools/run-helm.sh,/milvus/tools/merge
        image: "milvusdb/milvus-config-tool:v0.1.1"
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /milvus/tools
          name: tools
      containers:
      - name: datanode
        image: "milvusdb/milvus:v2.2.11"
        imagePullPolicy: IfNotPresent
        args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "datanode" ]
        env:
        ports:
          - name: datanode
            containerPort: 21124
            protocol: TCP
          - name: metrics
            containerPort: 9091
            protocol: TCP
        livenessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 30
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        resources:
          {}
        volumeMounts:
        - name: milvus-config
          mountPath: /milvus/configs/default.yaml
          subPath: default.yaml
          readOnly: true
        - name: milvus-config
          mountPath: /milvus/configs/user.yaml
          subPath: user.yaml
          readOnly: true
        - mountPath: /milvus/tools
          name: tools
      volumes:
      - name: milvus-config
        configMap:
          name: my-milvus-milvus
      - name: tools
        emptyDir: {}
---
# Source: milvus/templates/indexcoord-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-indexcoord
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "indexcoord"

  annotations:

spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "indexcoord"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus
        component: "indexcoord"

      annotations:
        checksum/config: b16c4c9fe32787486b1ac5c88467f7e3ddb64d9e0f97e9f9b827434867dc474c

    spec:
      serviceAccountName: default
      initContainers:
      - name: config
        command:
        - /cp
        - /run-helm.sh,/merge
        - /milvus/tools/run-helm.sh,/milvus/tools/merge
        image: "milvusdb/milvus-config-tool:v0.1.1"
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /milvus/tools
          name: tools
      containers:
      - name: indexcoord
        image: "milvusdb/milvus:v2.2.11"
        imagePullPolicy: IfNotPresent
        args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "indexcoord" ]
        env:
        ports:
          - name: indexcoord
            containerPort: 31000
            protocol: TCP
          - name: metrics
            containerPort: 9091
            protocol: TCP
        livenessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 30
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        resources:
          {}
        volumeMounts:
        - name: milvus-config
          mountPath: /milvus/configs/default.yaml
          subPath: default.yaml
          readOnly: true
        - name: milvus-config
          mountPath: /milvus/configs/user.yaml
          subPath: user.yaml
          readOnly: true
        - mountPath: /milvus/tools
          name: tools

      volumes:
      - name: milvus-config
        configMap:
          name: my-milvus-milvus
      - name: tools
        emptyDir: {}
---
# Source: milvus/templates/indexnode-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-indexnode
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "indexnode"

  annotations:

spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "indexnode"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus
        component: "indexnode"

      annotations:
        checksum/config: b16c4c9fe32787486b1ac5c88467f7e3ddb64d9e0f97e9f9b827434867dc474c

    spec:
      serviceAccountName: default
      initContainers:
      - name: config
        command:
        - /cp
        - /run-helm.sh,/merge
        - /milvus/tools/run-helm.sh,/milvus/tools/merge
        image: "milvusdb/milvus-config-tool:v0.1.1"
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /milvus/tools
          name: tools
      containers:
      - name: indexnode
        image: "milvusdb/milvus:v2.2.11"
        imagePullPolicy: IfNotPresent
        args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "indexnode" ]
        env:
        ports:
          - name: indexnode
            containerPort: 21121
            protocol: TCP
          - name: metrics
            containerPort: 9091
            protocol: TCP
        livenessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 30
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        resources:
          {}
        volumeMounts:
        - name: milvus-config
          mountPath: /milvus/configs/default.yaml
          subPath: default.yaml
          readOnly: true
        - name: milvus-config
          mountPath: /milvus/configs/user.yaml
          subPath: user.yaml
          readOnly: true
        - mountPath: /milvus/tools
          name: tools
        - mountPath: /milvus/data
          name: disk

      volumes:
      - name: milvus-config
        configMap:
          name: my-milvus-milvus
      - name: tools
        emptyDir: {}
      - name: disk
        emptyDir: {}
---
# Source: milvus/templates/proxy-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-proxy
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "proxy"

  annotations:

spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "proxy"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus
        component: "proxy"

      annotations:
        checksum/config: b16c4c9fe32787486b1ac5c88467f7e3ddb64d9e0f97e9f9b827434867dc474c

    spec:
      serviceAccountName: default
      initContainers:
      - name: config
        command:
        - /cp
        - /run-helm.sh,/merge
        - /milvus/tools/run-helm.sh,/milvus/tools/merge
        image: "milvusdb/milvus-config-tool:v0.1.1"
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /milvus/tools
          name: tools
      containers:
      - name: proxy
        image: "milvusdb/milvus:v2.2.11"
        imagePullPolicy: IfNotPresent
        args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "proxy" ]
        env:
        ports:
          - name: milvus
            containerPort: 19530
            protocol: TCP
          - name: metrics
            containerPort: 9091
            protocol: TCP
        livenessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 30
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        resources:
          {}
        volumeMounts:
        - name: milvus-config
          mountPath: /milvus/configs/default.yaml
          subPath: default.yaml
          readOnly: true
        - name: milvus-config
          mountPath: /milvus/configs/user.yaml
          subPath: user.yaml
          readOnly: true
        - mountPath: /milvus/tools
          name: tools

      volumes:
      - name: milvus-config
        configMap:
          name: my-milvus-milvus
      - name: tools
        emptyDir: {}
---
# Source: milvus/templates/querycoord-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-querycoord
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "querycoord"

  annotations:

spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "querycoord"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus

        component: "querycoord"
      annotations:
        checksum/config: b16c4c9fe32787486b1ac5c88467f7e3ddb64d9e0f97e9f9b827434867dc474c

    spec:
      serviceAccountName: default
      initContainers:
      - name: config
        command:
        - /cp
        - /run-helm.sh,/merge
        - /milvus/tools/run-helm.sh,/milvus/tools/merge
        image: "milvusdb/milvus-config-tool:v0.1.1"
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /milvus/tools
          name: tools
      containers:
      - name: querycoord
        image: "milvusdb/milvus:v2.2.11"
        imagePullPolicy: IfNotPresent
        args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "querycoord" ]
        env:
        ports:
          - name: querycoord
            containerPort: 19531
            protocol: TCP
          - name: metrics
            containerPort: 9091
            protocol: TCP
        livenessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 30
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        resources:
          {}
        volumeMounts:
        - name: milvus-config
          mountPath: /milvus/configs/default.yaml
          subPath: default.yaml
          readOnly: true
        - name: milvus-config
          mountPath: /milvus/configs/user.yaml
          subPath: user.yaml
          readOnly: true
        - mountPath: /milvus/tools
          name: tools

      volumes:
      - name: milvus-config
        configMap:
          name: my-milvus-milvus
      - name: tools
        emptyDir: {}
---
# Source: milvus/templates/querynode-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-querynode
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "querynode"

  annotations:

spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "querynode"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus
        component: "querynode"

      annotations:
        checksum/config: b16c4c9fe32787486b1ac5c88467f7e3ddb64d9e0f97e9f9b827434867dc474c

    spec:
      serviceAccountName: default
      initContainers:
      - name: config
        command:
        - /cp
        - /run-helm.sh,/merge
        - /milvus/tools/run-helm.sh,/milvus/tools/merge
        image: "milvusdb/milvus-config-tool:v0.1.1"
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /milvus/tools
          name: tools
      containers:
      - name: querynode
        image: "milvusdb/milvus:v2.2.11"
        imagePullPolicy: IfNotPresent
        args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "querynode" ]
        env:
        ports:
          - name: querynode
            containerPort: 21123
            protocol: TCP
          - name: metrics
            containerPort: 9091
            protocol: TCP
        livenessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 30
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        resources:
          {}
        volumeMounts:
        - name: milvus-config
          mountPath: /milvus/configs/default.yaml
          subPath: default.yaml
          readOnly: true
        - name: milvus-config
          mountPath: /milvus/configs/user.yaml
          subPath: user.yaml
          readOnly: true
        - mountPath: /milvus/tools
          name: tools
        - mountPath: /milvus/data
          name: disk

      volumes:
      - name: milvus-config
        configMap:
          name: my-milvus-milvus
      - name: tools
        emptyDir: {}
      - name: disk
        emptyDir: {}
---
# Source: milvus/templates/rootcoord-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-milvus-milvus-rootcoord
  labels:
    helm.sh/chart: milvus-4.0.11
    app.kubernetes.io/name: milvus
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/version: "2.2.3"
    app.kubernetes.io/managed-by: Helm
    component: "rootcoord"

  annotations:

spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: milvus
      app.kubernetes.io/instance: my-milvus
      component: "rootcoord"
  template:
    metadata:
      labels:
        app.kubernetes.io/name: milvus
        app.kubernetes.io/instance: my-milvus
        component: "rootcoord"

      annotations:
        checksum/config: b16c4c9fe32787486b1ac5c88467f7e3ddb64d9e0f97e9f9b827434867dc474c

    spec:
      serviceAccountName: default
      initContainers:
      - name: config
        command:
        - /cp
        - /run-helm.sh,/merge
        - /milvus/tools/run-helm.sh,/milvus/tools/merge
        image: "milvusdb/milvus-config-tool:v0.1.1"
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /milvus/tools
          name: tools
      containers:
      - name: rootcoord
        image: "milvusdb/milvus:v2.2.11"
        imagePullPolicy: IfNotPresent
        args: [ "/milvus/tools/run-helm.sh", "milvus", "run", "rootcoord" ]
        env:
        ports:
          - name: rootcoord
            containerPort: 53100
            protocol: TCP
          - name: metrics
            containerPort: 9091
            protocol: TCP
        livenessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 30
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 90
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        resources:
          {}
        volumeMounts:
        - name: milvus-config
          mountPath: /milvus/configs/default.yaml
          subPath: default.yaml
          readOnly: true
        - name: milvus-config
          mountPath: /milvus/configs/user.yaml
          subPath: user.yaml
          readOnly: true
        - mountPath: /milvus/tools
          name: tools

      volumes:
      - name: milvus-config
        configMap:
          name: my-milvus-milvus
      - name: tools
        emptyDir: {}
---
# Source: milvus/charts/etcd/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: my-milvus-etcd
  namespace: default
  labels:
    app.kubernetes.io/name: etcd
    helm.sh/chart: etcd-6.3.3
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
spec:
  replicas: 3
  selector:
    matchLabels:
      app.kubernetes.io/name: etcd
      app.kubernetes.io/instance: my-milvus
  serviceName: my-milvus-etcd-headless
  podManagementPolicy: Parallel
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app.kubernetes.io/name: etcd
        helm.sh/chart: etcd-6.3.3
        app.kubernetes.io/instance: my-milvus
        app.kubernetes.io/managed-by: Helm
      annotations:
    spec:

      affinity:
        podAffinity:

        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app.kubernetes.io/name: etcd
                    app.kubernetes.io/instance: my-milvus
                namespaces:
                  - "default"
                topologyKey: kubernetes.io/hostname
              weight: 1
        nodeAffinity:

      securityContext:
        fsGroup: 1001
      serviceAccountName: "default"
      containers:
        - name: etcd
          image: docker.io/milvusdb/etcd:3.5.5-r2
          imagePullPolicy: "IfNotPresent"
          securityContext:
            runAsNonRoot: true
            runAsUser: 1001
          env:
            - name: BITNAMI_DEBUG
              value: "false"
            - name: MY_POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: MY_POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: ETCDCTL_API
              value: "3"
            - name: ETCD_ON_K8S
              value: "yes"
            - name: ETCD_START_FROM_SNAPSHOT
              value: "no"
            - name: ETCD_DISASTER_RECOVERY
              value: "no"
            - name: ETCD_NAME
              value: "$(MY_POD_NAME)"
            - name: ETCD_DATA_DIR
              value: "/bitnami/etcd/data"
            - name: ETCD_LOG_LEVEL
              value: "info"
            - name: ALLOW_NONE_AUTHENTICATION
              value: "yes"
            - name: ETCD_ADVERTISE_CLIENT_URLS
              value: "http://$(MY_POD_NAME).my-milvus-etcd-headless.default.svc.cluster.local:2379"
            - name: ETCD_LISTEN_CLIENT_URLS
              value: "http://0.0.0.0:2379"
            - name: ETCD_INITIAL_ADVERTISE_PEER_URLS
              value: "http://$(MY_POD_NAME).my-milvus-etcd-headless.default.svc.cluster.local:2380"
            - name: ETCD_LISTEN_PEER_URLS
              value: "http://0.0.0.0:2380"
            - name: ETCD_AUTO_COMPACTION_MODE
              value: "revision"
            - name: ETCD_AUTO_COMPACTION_RETENTION
              value: "1000"
            - name: ETCD_INITIAL_CLUSTER_TOKEN
              value: "etcd-cluster-k8s"
            - name: ETCD_INITIAL_CLUSTER_STATE
              value: "new"
            - name: ETCD_INITIAL_CLUSTER
              value: "my-milvus-etcd-0=http://my-milvus-etcd-0.my-milvus-etcd-headless.default.svc.cluster.local:2380,my-milvus-etcd-1=http://my-milvus-etcd-1.my-milvus-etcd-headless.default.svc.cluster.local:2380,my-milvus-etcd-2=http://my-milvus-etcd-2.my-milvus-etcd-headless.default.svc.cluster.local:2380"
            - name: ETCD_CLUSTER_DOMAIN
              value: "my-milvus-etcd-headless.default.svc.cluster.local"
            - name: ETCD_QUOTA_BACKEND_BYTES
              value: "4294967296"
            - name: ETCD_HEARTBEAT_INTERVAL
              value: "500"
            - name: ETCD_ELECTION_TIMEOUT
              value: "2500"
          envFrom:
          ports:
            - name: client
              containerPort: 2379
              protocol: TCP
            - name: peer
              containerPort: 2380
              protocol: TCP
          livenessProbe:
            exec:
              command:
                - /opt/bitnami/scripts/etcd/healthcheck.sh
            initialDelaySeconds: 60
            periodSeconds: 30
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 5
          readinessProbe:
            exec:
              command:
                - /opt/bitnami/scripts/etcd/healthcheck.sh
            initialDelaySeconds: 60
            periodSeconds: 10
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 5
          lifecycle:
            preStop:
              exec:
                command:
                  - /opt/bitnami/scripts/etcd/prestop.sh
          resources:
            limits: {}
            requests: {}
          volumeMounts:
            - name: data
              mountPath: /bitnami/etcd
      volumes:
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        storageClassName: nfs-storage
        accessModes:
          - "ReadWriteOnce"
        resources:
          requests:
            storage: "10Gi"
---
# Source: milvus/charts/kafka/charts/zookeeper/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: my-milvus-zookeeper
  namespace: default
  labels:
    app.kubernetes.io/name: zookeeper
    helm.sh/chart: zookeeper-8.1.2
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: zookeeper
    role: zookeeper
spec:
  replicas: 3
  podManagementPolicy: Parallel
  selector:
    matchLabels:
      app.kubernetes.io/name: zookeeper
      app.kubernetes.io/instance: my-milvus
      app.kubernetes.io/component: zookeeper
  serviceName: my-milvus-zookeeper-headless
  updateStrategy:
    rollingUpdate: {}
    type: RollingUpdate
  template:
    metadata:
      annotations:
      labels:
        app.kubernetes.io/name: zookeeper
        helm.sh/chart: zookeeper-8.1.2
        app.kubernetes.io/instance: my-milvus
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: zookeeper
    spec:
      serviceAccountName: default

      affinity:
        podAffinity:

        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app.kubernetes.io/name: zookeeper
                    app.kubernetes.io/instance: my-milvus
                    app.kubernetes.io/component: zookeeper
                namespaces:
                  - "default"
                topologyKey: kubernetes.io/hostname
              weight: 1
        nodeAffinity:

      securityContext:
        fsGroup: 1001
      initContainers:
      containers:
        - name: zookeeper
          image: docker.io/bitnami/zookeeper:3.7.0-debian-10-r320
          imagePullPolicy: "IfNotPresent"
          securityContext:
            runAsNonRoot: true
            runAsUser: 1001
          command:
            - /scripts/setup.sh
          resources:
            limits: {}
            requests:
              cpu: 250m
              memory: 256Mi
          env:
            - name: BITNAMI_DEBUG
              value: "false"
            - name: ZOO_DATA_LOG_DIR
              value: ""
            - name: ZOO_PORT_NUMBER
              value: "2181"
            - name: ZOO_TICK_TIME
              value: "2000"
            - name: ZOO_INIT_LIMIT
              value: "10"
            - name: ZOO_SYNC_LIMIT
              value: "5"
            - name: ZOO_PRE_ALLOC_SIZE
              value: "65536"
            - name: ZOO_SNAPCOUNT
              value: "100000"
            - name: ZOO_MAX_CLIENT_CNXNS
              value: "60"
            - name: ZOO_4LW_COMMANDS_WHITELIST
              value: "srvr, mntr, ruok"
            - name: ZOO_LISTEN_ALLIPS_ENABLED
              value: "no"
            - name: ZOO_AUTOPURGE_INTERVAL
              value: "0"
            - name: ZOO_AUTOPURGE_RETAIN_COUNT
              value: "3"
            - name: ZOO_MAX_SESSION_TIMEOUT
              value: "40000"
            - name: ZOO_SERVERS
              value: my-milvus-zookeeper-0.my-milvus-zookeeper-headless.default.svc.cluster.local:2888:3888::1 my-milvus-zookeeper-1.my-milvus-zookeeper-headless.default.svc.cluster.local:2888:3888::2 my-milvus-zookeeper-2.my-milvus-zookeeper-headless.default.svc.cluster.local:2888:3888::3 
            - name: ZOO_ENABLE_AUTH
              value: "no"
            - name: ZOO_HEAP_SIZE
              value: "1024"
            - name: ZOO_LOG_LEVEL
              value: "ERROR"
            - name: ALLOW_ANONYMOUS_LOGIN
              value: "yes"
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
          ports:
            - name: client
              containerPort: 2181
            - name: follower
              containerPort: 2888
            - name: election
              containerPort: 3888
          livenessProbe:
            failureThreshold: 6
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
            exec:
              command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok']
          readinessProbe:
            failureThreshold: 6
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
            exec:
              command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok']
          volumeMounts:
            - name: scripts
              mountPath: /scripts/setup.sh
              subPath: setup.sh
            - name: data
              mountPath: /bitnami/zookeeper
      volumes:
        - name: scripts
          configMap:
            name: my-milvus-zookeeper-scripts
            defaultMode: 0755
  volumeClaimTemplates:
    - metadata:
        name: data
        annotations:
      spec:
        storageClassName: nfs-storage
        accessModes:
          - "ReadWriteOnce"
        resources:
          requests:
            storage: "8Gi"
---
# Source: milvus/charts/kafka/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: my-milvus-kafka
  namespace: "default"
  labels:
    app.kubernetes.io/name: kafka
    helm.sh/chart: kafka-15.5.1
    app.kubernetes.io/instance: my-milvus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: kafka
spec:
  podManagementPolicy: Parallel
  replicas: 3
  selector:
    matchLabels:
      app.kubernetes.io/name: kafka
      app.kubernetes.io/instance: my-milvus
      app.kubernetes.io/component: kafka
  serviceName: my-milvus-kafka-headless
  updateStrategy:
    rollingUpdate: {}
    type: RollingUpdate
  template:
    metadata:
      labels:
        app.kubernetes.io/name: kafka
        helm.sh/chart: kafka-15.5.1
        app.kubernetes.io/instance: my-milvus
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: kafka
      annotations:
    spec:

      hostNetwork: false
      hostIPC: false
      affinity:
        podAffinity:

        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app.kubernetes.io/name: kafka
                    app.kubernetes.io/instance: my-milvus
                    app.kubernetes.io/component: kafka
                namespaces:
                  - "default"
                topologyKey: kubernetes.io/hostname
              weight: 1
        nodeAffinity:

      terminationGracePeriodSeconds: 90
      securityContext:
        fsGroup: 1001
      serviceAccountName: my-milvus-kafka
      containers:
        - name: kafka
          image: docker.io/bitnami/kafka:3.1.0-debian-10-r52
          imagePullPolicy: "IfNotPresent"
          securityContext:
            runAsNonRoot: true
            runAsUser: 1001
          command:
            - /scripts/setup.sh
          env:
            - name: BITNAMI_DEBUG
              value: "false"
            - name: MY_POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: MY_POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: KAFKA_CFG_ZOOKEEPER_CONNECT
              value: "my-milvus-zookeeper"
            - name: KAFKA_INTER_BROKER_LISTENER_NAME
              value: "INTERNAL"
            - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
              value: "INTERNAL:PLAINTEXT,CLIENT:PLAINTEXT"
            - name: KAFKA_CFG_LISTENERS
              value: "INTERNAL://:9093,CLIENT://:9092"
            - name: KAFKA_CFG_ADVERTISED_LISTENERS
              value: "INTERNAL://$(MY_POD_NAME).my-milvus-kafka-headless.default.svc.cluster.local:9093,CLIENT://$(MY_POD_NAME).my-milvus-kafka-headless.default.svc.cluster.local:9092"
            - name: ALLOW_PLAINTEXT_LISTENER
              value: "yes"
            - name: KAFKA_VOLUME_DIR
              value: "/bitnami/kafka"
            - name: KAFKA_LOG_DIR
              value: "/opt/bitnami/kafka/logs"
            - name: KAFKA_CFG_DELETE_TOPIC_ENABLE
              value: "false"
            - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE
              value: "true"
            - name: KAFKA_HEAP_OPTS
              value: "-Xmx4096m -Xms4096m"
            - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MESSAGES
              value: "10000"
            - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MS
              value: "1000"
            - name: KAFKA_CFG_LOG_RETENTION_BYTES
              value: "-1"
            - name: KAFKA_CFG_LOG_RETENTION_CHECK_INTERVALS_MS
              value: "300000"
            - name: KAFKA_CFG_LOG_RETENTION_HOURS
              value: "168"
            - name: KAFKA_CFG_MESSAGE_MAX_BYTES
              value: "10485760"
            - name: KAFKA_CFG_LOG_SEGMENT_BYTES
              value: "1073741824"
            - name: KAFKA_CFG_LOG_DIRS
              value: "/bitnami/kafka/data"
            - name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR
              value: "3"
            - name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR
              value: "3"
            - name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
              value: "1"
            - name: KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR
              value: "1"
            - name: KAFKA_CFG_NUM_IO_THREADS
              value: "8"
            - name: KAFKA_CFG_NUM_NETWORK_THREADS
              value: "3"
            - name: KAFKA_CFG_NUM_PARTITIONS
              value: "1"
            - name: KAFKA_CFG_NUM_RECOVERY_THREADS_PER_DATA_DIR
              value: "1"
            - name: KAFKA_CFG_SOCKET_RECEIVE_BUFFER_BYTES
              value: "102400"
            - name: KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES
              value: "104857600"
            - name: KAFKA_CFG_SOCKET_SEND_BUFFER_BYTES
              value: "102400"
            - name: KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS
              value: "6000"
            - name: KAFKA_CFG_AUTHORIZER_CLASS_NAME
              value: ""
            - name: KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND
              value: "true"
            - name: KAFKA_CFG_SUPER_USERS
              value: "User:admin"
            - name: KAFKA_CFG_MAX_PARTITION_FETCH_BYTES
              value: "5242880"
            - name: KAFKA_CFG_MAX_REQUEST_SIZE
              value: "5242880"
            - name: KAFKA_CFG_REPLICA_FETCH_MAX_BYTES
              value: "10485760"
            - name: KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES
              value: "5242880"
            - name: KAFKA_CFG_LOG_ROLL_HOURS
              value: "24"
          ports:
            - name: kafka-client
              containerPort: 9092
            - name: kafka-internal
              containerPort: 9093
          livenessProbe:
            failureThreshold: 3
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
            tcpSocket:
              port: kafka-client
          readinessProbe:
            failureThreshold: 6
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
            tcpSocket:
              port: kafka-client
          startupProbe:
            failureThreshold: 15
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
            tcpSocket:
              port: kafka-client
          resources:
            limits: {}
            requests: {}
          volumeMounts:
            - name: data
              mountPath: /bitnami/kafka
            - name: logs
              mountPath: /opt/bitnami/kafka/logs
            - name: scripts
              mountPath: /scripts/setup.sh
              subPath: setup.sh
      volumes:
        - name: scripts
          configMap:
            name: my-milvus-kafka-scripts
            defaultMode: 0755
        - name: logs
          emptyDir: {}
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        storageClassName: nfs-storage
        accessModes:
          - "ReadWriteOnce"
        resources:
          requests:
            storage: "300Gi"
---
# Source: milvus/charts/minio/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: my-milvus-minio
  labels:
    app: minio
    chart: minio-8.0.15
    release: my-milvus
    heritage: Helm
spec:
  updateStrategy:
    type: RollingUpdate
  podManagementPolicy: "Parallel"
  serviceName: my-milvus-minio-svc
  replicas: 4
  selector:
    matchLabels:
      app: minio
      release: my-milvus
  template:
    metadata:
      name: my-milvus-minio
      labels:
        app: minio
        release: my-milvus
      annotations:
        checksum/secrets: c16a3f8ad12ac5f687f46f6445a7074c6fe3d9c2780fd37cd94442c9a39686fc
        checksum/config: 4bd8f07c5475291eae8cd09acdc370ab9ae2b161296ec7b19c5aaae933b9e28d
    spec:
      serviceAccountName: "my-milvus-minio"
      securityContext:
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
      containers:
        - name: minio
          image: minio/minio:RELEASE.2023-03-20T20-16-18Z
          imagePullPolicy: IfNotPresent

          command: [ "/bin/sh",
            "-ce",
            "/usr/bin/docker-entrypoint.sh minio -S /etc/minio/certs/ server  http://my-milvus-minio-{0...3}.my-milvus-minio-svc.default.svc.cluster.local/export" ]
          volumeMounts:
            - name: export
              mountPath: /export            
          ports:
            - name: http
              containerPort: 9000
          livenessProbe:
            httpGet:
              path: /minio/health/live
              port: http
              scheme: HTTP
            initialDelaySeconds: 5
            periodSeconds: 5
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 5
          readinessProbe:
            tcpSocket:
              port: http
            initialDelaySeconds: 5
            periodSeconds: 5
            timeoutSeconds: 1
            successThreshold: 1
            failureThreshold: 5
          startupProbe:
            tcpSocket:
              port: http
            initialDelaySeconds: 0
            periodSeconds: 10
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 60
          env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: my-milvus-minio
                  key: accesskey
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: my-milvus-minio
                  key: secretkey
          resources:
            requests:
              memory: 2Gi      
      volumes:
        - name: minio-user
          secret:
            secretName: my-milvus-minio        
  volumeClaimTemplates:
    - metadata:
        name: export
      spec:
        storageClassName: nfs-storage
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 500Gi

Milvus Log

No response

Anything else?

No response

[hf185]

I modified the KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS parameter in kafka, and now kafka can start successfully, but the pod of milvus cannot start the error message:

[hf185]

In addition, I found that etcd seems to have errors

[hf185]

@yanliang567

[yanliang567]

/assign @locustbaby could you please help on this issue?

/unassign

[hf185]

Thank you, I solved it by myself, there is a problem with k8s dns resolution, just restart coredns

[stale[bot]]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions. Rotten issues close after 30d of inactivity. Reopen the issue with /reopen.