search

apecloud / kubeblocks

KubeBlocks is an open-source control plane software that runs and manages databases, message queues and other stateful applications on K8s.
https://kubeblocks.io
GNU Affero General Public License v3.0
1.82k stars 158 forks source link

[BUG]orioledb pod restart after upgrade kb from 0.7.2 to 0.8 #6701

Open ahjing99 opened 4 months ago

ahjing99 commented 4 months ago
  1. Install kb 0.7.2, create cluster
  2. upgrade kb to 0.8.2-beta.10
  3. orioledb restart 
    
orioledb-tfvbyd-orioledb-0                                 0/5     ContainerCreating   0              5s
orioledb-tfvbyd-orioledb-1                                 5/5     Running             0              77s

➜ ~ k get rsm orioledb-tfvbyd-orioledb -o yaml apiVersion: workloads.kubeblocks.io/v1alpha1 kind: ReplicatedStateMachine metadata: annotations: config.kubeblocks.io/tpl-agamotto-configuration: orioledb-tfvbyd-orioledb-agamotto-configuration config.kubeblocks.io/tpl-etcd-env: orioledb-tfvbyd-orioledb-etcd-env config.kubeblocks.io/tpl-orioledb-configuration: orioledb-tfvbyd-orioledb-orioledb-configuration config.kubeblocks.io/tpl-orioledb-custom-metrics: orioledb-tfvbyd-orioledb-orioledb-custom-metrics config.kubeblocks.io/tpl-orioledb-pgbouncer-configuration: orioledb-tfvbyd-orioledb-orioledb-pgbouncer-configuration config.kubeblocks.io/tpl-orioledb-scripts: orioledb-tfvbyd-orioledb-orioledb-scripts kubeblocks.io/generation: "1" monitor.kubeblocks.io/agamotto.headless.rsm: "false" monitor.kubeblocks.io/scrape.headless.rsm: "false" creationTimestamp: "2024-02-28T02:56:02Z" finalizers:

➜ ~ k get sts orioledb-tfvbyd-orioledb -o yaml apiVersion: apps/v1 kind: StatefulSet metadata: annotations: config.kubeblocks.io/tpl-agamotto-configuration: orioledb-tfvbyd-orioledb-agamotto-configuration config.kubeblocks.io/tpl-etcd-env: orioledb-tfvbyd-orioledb-etcd-env config.kubeblocks.io/tpl-orioledb-configuration: orioledb-tfvbyd-orioledb-orioledb-configuration config.kubeblocks.io/tpl-orioledb-custom-metrics: orioledb-tfvbyd-orioledb-orioledb-custom-metrics config.kubeblocks.io/tpl-orioledb-pgbouncer-configuration: orioledb-tfvbyd-orioledb-orioledb-pgbouncer-configuration config.kubeblocks.io/tpl-orioledb-scripts: orioledb-tfvbyd-orioledb-orioledb-scripts kubeblocks.io/generation: "1" creationTimestamp: "2024-02-28T02:56:03Z" finalizers:

ahjing99 commented 4 months ago 
➜  ~ k get cd orioledb -o yaml
apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterDefinition
metadata:
  annotations:
    meta.helm.sh/release-name: kb-addon-orioledb
    meta.helm.sh/release-namespace: kb-system
  creationTimestamp: "2024-02-28T02:55:23Z"
  finalizers:
  - clusterdefinition.kubeblocks.io/finalizer
  generation: 2
  labels:
    app.kubernetes.io/instance: kb-addon-orioledb
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: orioledb
    app.kubernetes.io/version: 14.7.2-beta1
    config.kubeblocks.io/constraints-orioledb-cc: orioledb-cc
    config.kubeblocks.io/tpl-agamotto-configuration: orioledb-agamotto-configuration
    config.kubeblocks.io/tpl-etcd-env: etcd-env
    config.kubeblocks.io/tpl-orioledb-configuration: orioledb-configuration
    config.kubeblocks.io/tpl-orioledb-custom-metrics: orioledb-custom-metrics
    config.kubeblocks.io/tpl-orioledb-pgbouncer-configuration: orioledb-pgbouncer-configuration
    config.kubeblocks.io/tpl-orioledb-scripts: orioledb-scripts
    helm.sh/chart: orioledb-0.7.2
  name: orioledb
  resourceVersion: "489638"
  uid: 9305bf83-bae8-4bf3-9af3-91d3c95de834
spec:
  componentDefs:
  - characterType: postgresql
    configSpecs:
    - constraintRef: orioledb-cc
      defaultMode: 511
      keys:
      - postgresql.conf
      name: orioledb-configuration
      namespace: kb-system
      templateRef: orioledb-configuration
      volumeName: postgresql-config
    - defaultMode: 511
      keys:
      - pgbouncer.ini
      name: orioledb-pgbouncer-configuration
      namespace: kb-system
      templateRef: orioledb-pgbouncer-configuration
      volumeName: pgbouncer-config
    - defaultMode: 511
      name: orioledb-custom-metrics
      namespace: kb-system
      templateRef: orioledb-custom-metrics
      volumeName: postgresql-custom-metrics
    - defaultMode: 511
      name: agamotto-configuration
      namespace: kb-system
      templateRef: orioledb-agamotto-configuration
      volumeName: agamotto-configuration
    - defaultMode: 511
      name: etcd-env
      namespace: kb-system
      templateRef: etcd-env
      volumeName: patroni-dependency
    customLabelSpecs:
    - key: apps.kubeblocks.postgres.patroni/scope
      resources:
      - gvk: v1/Pod
        selector:
          app.kubernetes.io/managed-by: kubeblocks
      - gvk: apps/v1/StatefulSet
        selector:
          app.kubernetes.io/managed-by: kubeblocks
      value: $(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-patroni$(KB_CLUSTER_UID_POSTFIX_8)
    logConfigs:
    - filePathPattern: /home/postgres/pgdata/pgroot/data/log/postgresql-*
      name: running
    monitor:
      builtIn: false
      exporterConfig:
        scrapePath: /metrics
        scrapePort: 9187
    name: orioledb
    podSpec:
      containers:
      - command:
        - bin/bash
        - -c
        - /kb-scripts/setup.sh
        env:
        - name: PGROOT
          value: /home/postgres/pgdata/pgroot
        - name: SCOPE
          value: $(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-patroni$(KB_CLUSTER_UID_POSTFIX_8)
        - name: KUBERNETES_SCOPE_LABEL
          value: apps.kubeblocks.postgres.patroni/scope
        - name: KUBERNETES_ROLE_LABEL
          value: apps.kubeblocks.postgres.patroni/role
        - name: RESTORE_DATA_DIR
          value: /home/postgres/pgdata/kb_restore
        - name: KB_PG_CONFIG_PATH
          value: /home/postgres/conf/postgresql.conf
        - name: ALLOW_NOSSL
          value: "true"
        - name: PGROOT
          value: /home/postgres/pgdata/pgroot
        - name: PGDATA
          value: /home/postgres/pgdata/pgroot/data
        - name: POD_IP
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.podIP
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: PGUSER_SUPERUSER
          valueFrom:
            secretKeyRef:
              key: username
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: PGPASSWORD_SUPERUSER
          valueFrom:
            secretKeyRef:
              key: password
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: PGUSER_ADMIN
          value: superadmin
        - name: PGPASSWORD_ADMIN
          valueFrom:
            secretKeyRef:
              key: password
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: PGUSER_STANDBY
          value: standby
        - name: PGPASSWORD_STANDBY
          valueFrom:
            secretKeyRef:
              key: password
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: PGUSER
          valueFrom:
            secretKeyRef:
              key: username
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              key: password
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        imagePullPolicy: IfNotPresent
        name: postgresql
        ports:
        - containerPort: 5432
          name: tcp-orioledb
          protocol: TCP
        - containerPort: 8008
          name: patroni
          protocol: TCP
        readinessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - -ee
            - |
              exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432
              [ -f /postgresql/tmp/.initialized ] || [ -f /postgresql/.initialized ]
          failureThreshold: 3
          initialDelaySeconds: 10
          periodSeconds: 30
          successThreshold: 1
          timeoutSeconds: 5
        resources: {}
        securityContext:
          runAsUser: 0
        volumeMounts:
        - mountPath: /dev/shm
          name: dshm
        - mountPath: /home/postgres/pgdata
          name: data
        - mountPath: /home/postgres/conf
          name: postgresql-config
        - mountPath: /kb-scripts
          name: scripts
        - mountPath: /kb-podinfo
          name: pod-info
        - mountPath: /dependency
          name: patroni-dependency
      - command:
        - /kb-scripts/pgbouncer_setup.sh
        env:
        - name: PGBOUNCER_AUTH_TYPE
          value: md5
        - name: POSTGRESQL_USERNAME
          valueFrom:
            secretKeyRef:
              key: username
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: POSTGRESQL_PASSWORD
          valueFrom:
            secretKeyRef:
              key: password
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: POSTGRESQL_PORT
          value: "5432"
        - name: POSTGRESQL_HOST
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.podIP
        - name: PGBOUNCER_PORT
          value: "6432"
        - name: PGBOUNCER_BIND_ADDRESS
          value: 0.0.0.0
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 3
          initialDelaySeconds: 15
          periodSeconds: 30
          successThreshold: 1
          tcpSocket:
            port: tcp-pgbouncer
          timeoutSeconds: 5
        name: pgbouncer
        ports:
        - containerPort: 6432
          name: tcp-pgbouncer
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          initialDelaySeconds: 15
          periodSeconds: 30
          successThreshold: 1
          tcpSocket:
            port: tcp-pgbouncer
          timeoutSeconds: 5
        resources: {}
        securityContext:
          runAsUser: 0
        volumeMounts:
        - mountPath: /home/pgbouncer/conf
          name: pgbouncer-config
        - mountPath: /kb-scripts
          name: scripts
      - command:
        - /bin/agamotto
        - --config=/opt/agamotto/agamotto-config.yaml
        env:
        - name: ENDPOINT
          value: 127.0.0.1:5432
        - name: DATA_SOURCE_PASS
          valueFrom:
            secretKeyRef:
              key: password
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: DATA_SOURCE_USER
          valueFrom:
            secretKeyRef:
              key: username
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        image: infracreate-registry.cn-zhangjiakou.cr.aliyuncs.com/apecloud/agamotto:0.1.0-beta.5
        imagePullPolicy: IfNotPresent
        name: metrics
        ports:
        - containerPort: 9187
          name: http-metrics
          protocol: TCP
        resources: {}
        securityContext:
          runAsUser: 0
        volumeMounts:
        - mountPath: /opt/conf
          name: postgresql-custom-metrics
        - mountPath: /opt/agamotto
          name: agamotto-configuration
      securityContext:
        fsGroup: 103
        runAsGroup: 103
        runAsUser: 0
      volumes:
      - emptyDir:
          medium: Memory
        name: dshm
      - downwardAPI:
          items:
          - fieldRef:
              fieldPath: metadata.labels['kubeblocks.io/role']
            path: pod-role
          - fieldRef:
              fieldPath: metadata.annotations['rs.apps.kubeblocks.io/primary']
            path: primary-pod
          - fieldRef:
              fieldPath: metadata.annotations['apps.kubeblocks.io/component-replicas']
            path: component-replicas
        name: pod-info
    probes:
      roleProbe:
        failureThreshold: 2
        periodSeconds: 1
        timeoutSeconds: 1
    scriptSpecs:
    - defaultMode: 511
      name: orioledb-scripts
      namespace: kb-system
      templateRef: orioledb-scripts
      volumeName: scripts
    service:
      ports:
      - name: tcp-orioledb
        port: 5432
        protocol: TCP
        targetPort: tcp-orioledb
      - name: tcp-pgbouncer
        port: 6432
        protocol: TCP
        targetPort: tcp-pgbouncer
    serviceRefDeclarations:
    - name: etcdService
      serviceRefDeclarationSpecs:
      - serviceKind: etcd
        serviceVersion: ^v3.\d...d$
    switchoverSpec:
      withCandidate:
        cmdExecutorConfig:
          args:
          - curl -s http://$(KB_REPLICATION_PRIMARY_POD_FQDN):8008/switchover -XPOST
            -d '{"leader":"$(KB_REPLICATION_PRIMARY_POD_NAME)","candidate":"$(KB_SWITCHOVER_CANDIDATE_NAME)"}'
          command:
          - /bin/bash
          - -c
          image: infracreate-registry.cn-zhangjiakou.cr.aliyuncs.com/apecloud/orioledb:beta1
      withoutCandidate:
        cmdExecutorConfig:
          args:
          - curl -s http://$(KB_REPLICATION_PRIMARY_POD_FQDN):8008/switchover -XPOST
            -d '{"leader":"$(KB_REPLICATION_PRIMARY_POD_NAME)"}'
          command:
          - /bin/bash
          - -c
          image: infracreate-registry.cn-zhangjiakou.cr.aliyuncs.com/apecloud/orioledb:beta1
    systemAccounts:
      accounts:
      - name: kbadmin
        provisionPolicy:
          scope: AnyPods
          statements:
            creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)';
            update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)';
          type: CreateByStmt
      - name: kbdataprotection
        provisionPolicy:
          scope: AnyPods
          statements:
            creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)';
            update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)';
          type: CreateByStmt
      - name: kbprobe
        provisionPolicy:
          scope: AnyPods
          statements:
            creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)';  GRANT pg_monitor
              TO $(USERNAME);
            update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)';
          type: CreateByStmt
      - name: kbmonitoring
        provisionPolicy:
          scope: AnyPods
          statements:
            creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)';  GRANT pg_monitor
              TO $(USERNAME);
            update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)';
          type: CreateByStmt
      - name: kbreplicator
        provisionPolicy:
          scope: AnyPods
          statements:
            creation: CREATE USER $(USERNAME) WITH REPLICATION PASSWORD '$(PASSWD)';
            update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)';
          type: CreateByStmt
      cmdExecutorConfig:
        args:
        - -h$(KB_ACCOUNT_ENDPOINT)
        - -c
        - $(KB_ACCOUNT_STATEMENT)
        command:
        - psql
        env:
        - name: PGUSER
          valueFrom:
            secretKeyRef:
              key: username
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        - name: PGPASSWORD
          valueFrom:
            secretKeyRef:
              key: password
              name: $(CONN_CREDENTIAL_SECRET_NAME)
              optional: false
        image: infracreate-registry.cn-zhangjiakou.cr.aliyuncs.com/apecloud/orioledb:beta1
      passwordConfig:
        length: 10
        letterCase: MixedCases
        numDigits: 5
        numSymbols: 0
    volumeTypes:
    - name: data
      type: data
    workloadType: Replication
  connectionCredential:
    endpoint: $(SVC_FQDN):$(SVC_PORT_tcp-orioledb)
    host: $(SVC_FQDN)
    password: $(RANDOM_PASSWD)
    port: $(SVC_PORT_tcp-orioledb)
    username: postgres
  type: orioledb
status:
  observedGeneration: 2
  phase: Available
github-actions[bot] commented 3 months ago

This issue has been marked as stale because it has been open for 30 days with no activity