➜ ~ kubectl get cluster asmysql-test-bk2
NAME CLUSTER-DEFINITION VERSION TERMINATION-POLICY STATUS AGE
asmysql-test-bk2 mysql WipeOut Creating 9m57s
➜ ~
➜ ~ kbcli cluster list-instances asmysql-test-bk2
NAME NAMESPACE CLUSTER COMPONENT STATUS ROLE ACCESSMODE AZ CPU(REQUEST/LIMIT) MEMORY(REQUEST/LIMIT) STORAGE NODE CREATED-TIME
asmysql-test-bk2-mysql-0 default asmysql-test-bk2 mysql Running cn-shanghai-b 100m / 100m 512Mi / 512Mi data:20Gi 172.16.0.2/172.16.0.2 Oct 16,2024 11:58 UTC+0800
asmysql-test-bk2-mysql-1 default asmysql-test-bk2 mysql Running cn-shanghai-b 100m / 100m 512Mi / 512Mi data:20Gi 172.16.0.72/172.16.0.72 Oct 16,2024 11:58 UTC+0800
➜ ~
log pod
kubectl logs asmysql-test-bk2-mysql-0
Defaulted container "mysql" out of: mysql, lorry, config-manager, init-data (init), init-xtrabackup (init), init-syncer (init)
2024-10-16T03:59:26Z INFO Initialize DB manager
2024-10-16T03:59:26Z DEBUG KB_WORKLOAD_TYPE ENV not set
2024-10-16T03:59:26Z INFO HTTPServer Starting HTTP Server
2024-10-16T03:59:26Z INFO HTTPServer API route path {"method": "POST", "path": ["/v1.0/stop", "/v1.0/switchover", "/v1.0/rebuild", "/v1.0/start"]}
2024-10-16T03:59:26Z INFO HTTPServer API route path {"method": "GET", "path": ["/v1.0/datasync"]}
2024-10-16T03:59:26Z INFO HTTPServer http server {"listen address": "0.0.0.0", "port": 3601}
2024-10-16T03:59:26Z INFO HA HA starting
2024-10-16T03:59:26Z INFO pinger Waiting for dns resolution to be ready
2024-10-16T03:59:27Z INFO pinger dns resolution is ready {"dns": "asmysql-test-bk2-mysql-0.asmysql-test-bk2-mysql-headless.default.svc"}
2024-10-16T03:59:27Z INFO DCS-K8S pod selector: app.kubernetes.io/instance=asmysql-test-bk2,app.kubernetes.io/managed-by=kubeblocks,apps.kubeblocks.io/component-name=mysql
2024-10-16T03:59:27Z INFO DCS-K8S podlist: 2
2024-10-16T03:59:27Z INFO DCS-K8S Leader configmap is not found {"configmap": "asmysql-test-bk2-mysql-leader"}
2024-10-16T03:59:27Z DEBUG HA cluster info {"cluster": {"ClusterCompName":"asmysql-test-bk2-mysql","Namespace":"default","Replicas":2,"HaConfig":{"DeleteMembers":null},"Leader":null,"Members":[{"Index":"","Name":"asmysql-test-bk2-mysql-0","Role":"","PodIP":"172.16.0.54","DBPort":"3306","SyncerPort":"3601","UID":"c9a117df-b8b4-4a96-80f9-c4a86cc257f1","ComponentName":"mysql","UseIP":false},{"Index":"","Name":"asmysql-test-bk2-mysql-1","Role":"","PodIP":"172.16.0.77","DBPort":"3306","SyncerPort":"3601","UID":"b9b70059-11a3-4193-9dfc-aa7a9b847a0e","ComponentName":"mysql","UseIP":false}],"Switchover":null,"Extra":null,"Resource":{"metadata":{"name":"asmysql-test-bk2","namespace":"default","uid":"ab492447-8c97-4b24-93e0-68d264ee43d2","resourceVersion":"541645","generation":1,"creationTimestamp":"2024-10-16T03:57:50Z","labels":{"clusterdefinition.kubeblocks.io/name":"mysql","clusterversion.kubeblocks.io/name":""},"annotations":{"kubeblocks.io/ops-request":"[{\"name\":\"asmysql-test-bk2\",\"type\":\"Restore\"}]","kubeblocks.io/restore-from-backup":"{\"mysql\":{\"doReadyRestoreAfterClusterRunning\":\"false\",\"name\":\"backup-default-asmysql-test-20241016115559\",\"namespace\":\"default\",\"volumeRestorePolicy\":\"Parallel\"}}"},"finalizers":["cluster.kubeblocks.io/finalizer"],"managedFields":[{"manager":"kubeblocks","operation":"Update","apiVersion":"apps.kubeblocks.io/v1alpha1","time":"2024-10-16T03:57:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubeblocks.io/ops-request":{},"f:kubeblocks.io/restore-from-backup":{}},"f:finalizers":{".":{},"v:\"cluster.kubeblocks.io/finalizer\"":{}},"f:labels":{".":{},"f:clusterdefinition.kubeblocks.io/name":{},"f:clusterversion.kubeblocks.io/name":{}}},"f:spec":{".":{},"f:clusterDefinitionRef":{},"f:componentSpecs":{},"f:resources":{".":{},"f:cpu":{},"f:memory":{}},"f:storage":{".":{},"f:size":{}},"f:terminationPolicy":{},"f:topology":{}}}},{"manager":"kubeblocks","operation":"Update","apiVersion":"apps.kubeblocks.io/v1alpha1","time":"2024-10-16T03:58:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{".":{},"f:clusterDefGeneration":{},"f:components":{".":{},"f:mysql":{".":{},"f:phase":{},"f:podsReady":{}}},"f:conditions":{},"f:observedGeneration":{},"f:phase":{}}},"subresource":"status"}]},"spec":{"clusterDefinitionRef":"mysql","terminationPolicy":"WipeOut","componentSpecs":[{"name":"mysql","componentDef":"mysql-8.0","replicas":2,"resources":{"limits":{"cpu":"100m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeClaimTemplates":[{"name":"data","spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"20Gi"}}}}]}],"resources":{"cpu":"0","memory":"0"},"storage":{"size":"0"},"monitor":{}},"status":{"observedGeneration":1,"phase":"Creating","components":{"mysql":{"phase":"Creating","podsReady":false}},"clusterDefGeneration":2,"conditions":[{"type":"ProvisioningStarted","status":"True","observedGeneration":1,"lastTransitionTime":"2024-10-16T03:57:50Z","reason":"PreCheckSucceed","message":"The operator has started the provisioning of Cluster: asmysql-test-bk2"},{"type":"ApplyResources","status":"True","observedGeneration":1,"lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ApplyResourcesSucceed","message":"Successfully applied for resources"},{"type":"ReplicasReady","status":"False","lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ReplicasNotReady","message":"pods are not ready in Components: [mysql], refer to related component message in Cluster.status.components"},{"type":"Ready","status":"False","lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ComponentsNotReady","message":"pods are unavailable in Components: [mysql], refer to related component message in Cluster.status.components"}]}}}}
2024-10-16T03:59:27Z INFO Hypervisor Starting Hypervisor
2024-10-16T03:59:27Z INFO Hypervisor Start DB Service {"command": "/usr/bin/bash -c mv /var/lib/mysql/plugin/audit_log.so /usr/lib64/mysql/plugin/\nrm -rf /var/lib/mysql/plugin\nchown -R mysql:root /var/lib/mysql\nskip_slave_start=\"OFF\"\nif [ -f /var/lib/mysql/data/.restore_new_cluster ]; then\n skip_slave_start=\"ON\"\nfi\n/scripts/mysql-entrypoint.sh\n"}
2024-10-16T03:59:27Z INFO MySQL there is no leader member
2024-10-16T03:59:27Z INFO HA check if DB Service is running
2024-10-16T03:59:27Z INFO Hypervisor Starting watcher on dbService
2024-10-16T03:59:27Z INFO MySQL DB is not ready {"error": "dial tcp 127.0.0.1:3306: connect: connection refused"}
== DB == 2024-10-16 03:59:28+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 8.0.38-1.el9 started.
== DB == 2024-10-16 03:59:29+00:00 [Note] [Entrypoint]: Switching to dedicated user 'mysql'
== DB == 2024-10-16 03:59:29+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 8.0.38-1.el9 started.
== DB == '/var/lib/mysql/mysql.sock' -> '/var/run/mysqld/mysqld.sock'
2024-10-16T03:59:32Z INFO MySQL DB is not ready {"error": "dial tcp 127.0.0.1:3306: connect: connection refused"}
2024-10-16T03:59:38Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:43Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:48Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:53Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:58Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T04:00:03Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T04:00:08Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
...
logs pod1
kubectl logs asmysql-test-bk2-mysql-1
Defaulted container "mysql" out of: mysql, lorry, config-manager, init-data (init), init-xtrabackup (init), init-syncer (init)
2024-10-16T03:59:19Z INFO Initialize DB manager
2024-10-16T03:59:19Z DEBUG KB_WORKLOAD_TYPE ENV not set
2024-10-16T03:59:19Z INFO HTTPServer Starting HTTP Server
2024-10-16T03:59:19Z INFO HTTPServer API route path {"method": "GET", "path": ["/v1.0/datasync"]}
2024-10-16T03:59:19Z INFO HTTPServer API route path {"method": "POST", "path": ["/v1.0/rebuild", "/v1.0/start", "/v1.0/stop", "/v1.0/switchover"]}
2024-10-16T03:59:19Z INFO HTTPServer http server {"listen address": "0.0.0.0", "port": 3601}
2024-10-16T03:59:19Z INFO HA HA starting
2024-10-16T03:59:19Z INFO pinger Waiting for dns resolution to be ready
2024-10-16T03:59:19Z INFO pinger dns resolution is ready {"dns": "asmysql-test-bk2-mysql-1.asmysql-test-bk2-mysql-headless.default.svc"}
2024-10-16T03:59:20Z INFO DCS-K8S pod selector: app.kubernetes.io/instance=asmysql-test-bk2,app.kubernetes.io/managed-by=kubeblocks,apps.kubeblocks.io/component-name=mysql
2024-10-16T03:59:20Z INFO DCS-K8S podlist: 2
2024-10-16T03:59:20Z INFO DCS-K8S Leader configmap is not found {"configmap": "asmysql-test-bk2-mysql-leader"}
2024-10-16T03:59:20Z DEBUG HA cluster info {"cluster": {"ClusterCompName":"asmysql-test-bk2-mysql","Namespace":"default","Replicas":2,"HaConfig":{"DeleteMembers":null},"Leader":null,"Members":[{"Index":"","Name":"asmysql-test-bk2-mysql-0","Role":"","PodIP":"172.16.0.54","DBPort":"3306","SyncerPort":"3601","UID":"c9a117df-b8b4-4a96-80f9-c4a86cc257f1","ComponentName":"mysql","UseIP":false},{"Index":"","Name":"asmysql-test-bk2-mysql-1","Role":"","PodIP":"172.16.0.77","DBPort":"3306","SyncerPort":"3601","UID":"b9b70059-11a3-4193-9dfc-aa7a9b847a0e","ComponentName":"mysql","UseIP":false}],"Switchover":null,"Extra":null,"Resource":{"metadata":{"name":"asmysql-test-bk2","namespace":"default","uid":"ab492447-8c97-4b24-93e0-68d264ee43d2","resourceVersion":"541645","generation":1,"creationTimestamp":"2024-10-16T03:57:50Z","labels":{"clusterdefinition.kubeblocks.io/name":"mysql","clusterversion.kubeblocks.io/name":""},"annotations":{"kubeblocks.io/ops-request":"[{\"name\":\"asmysql-test-bk2\",\"type\":\"Restore\"}]","kubeblocks.io/restore-from-backup":"{\"mysql\":{\"doReadyRestoreAfterClusterRunning\":\"false\",\"name\":\"backup-default-asmysql-test-20241016115559\",\"namespace\":\"default\",\"volumeRestorePolicy\":\"Parallel\"}}"},"finalizers":["cluster.kubeblocks.io/finalizer"],"managedFields":[{"manager":"kubeblocks","operation":"Update","apiVersion":"apps.kubeblocks.io/v1alpha1","time":"2024-10-16T03:57:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubeblocks.io/ops-request":{},"f:kubeblocks.io/restore-from-backup":{}},"f:finalizers":{".":{},"v:\"cluster.kubeblocks.io/finalizer\"":{}},"f:labels":{".":{},"f:clusterdefinition.kubeblocks.io/name":{},"f:clusterversion.kubeblocks.io/name":{}}},"f:spec":{".":{},"f:clusterDefinitionRef":{},"f:componentSpecs":{},"f:resources":{".":{},"f:cpu":{},"f:memory":{}},"f:storage":{".":{},"f:size":{}},"f:terminationPolicy":{},"f:topology":{}}}},{"manager":"kubeblocks","operation":"Update","apiVersion":"apps.kubeblocks.io/v1alpha1","time":"2024-10-16T03:58:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{".":{},"f:clusterDefGeneration":{},"f:components":{".":{},"f:mysql":{".":{},"f:phase":{},"f:podsReady":{}}},"f:conditions":{},"f:observedGeneration":{},"f:phase":{}}},"subresource":"status"}]},"spec":{"clusterDefinitionRef":"mysql","terminationPolicy":"WipeOut","componentSpecs":[{"name":"mysql","componentDef":"mysql-8.0","replicas":2,"resources":{"limits":{"cpu":"100m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeClaimTemplates":[{"name":"data","spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"20Gi"}}}}]}],"resources":{"cpu":"0","memory":"0"},"storage":{"size":"0"},"monitor":{}},"status":{"observedGeneration":1,"phase":"Creating","components":{"mysql":{"phase":"Creating","podsReady":false}},"clusterDefGeneration":2,"conditions":[{"type":"ProvisioningStarted","status":"True","observedGeneration":1,"lastTransitionTime":"2024-10-16T03:57:50Z","reason":"PreCheckSucceed","message":"The operator has started the provisioning of Cluster: asmysql-test-bk2"},{"type":"ApplyResources","status":"True","observedGeneration":1,"lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ApplyResourcesSucceed","message":"Successfully applied for resources"},{"type":"ReplicasReady","status":"False","lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ReplicasNotReady","message":"pods are not ready in Components: [mysql], refer to related component message in Cluster.status.components"},{"type":"Ready","status":"False","lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ComponentsNotReady","message":"pods are unavailable in Components: [mysql], refer to related component message in Cluster.status.components"}]}}}}
2024-10-16T03:59:20Z INFO Hypervisor Starting Hypervisor
2024-10-16T03:59:20Z INFO Hypervisor Start DB Service {"command": "/usr/bin/bash -c mv /var/lib/mysql/plugin/audit_log.so /usr/lib64/mysql/plugin/\nrm -rf /var/lib/mysql/plugin\nchown -R mysql:root /var/lib/mysql\nskip_slave_start=\"OFF\"\nif [ -f /var/lib/mysql/data/.restore_new_cluster ]; then\n skip_slave_start=\"ON\"\nfi\n/scripts/mysql-entrypoint.sh\n"}
2024-10-16T03:59:20Z INFO MySQL there is no leader member
2024-10-16T03:59:20Z INFO HA check if DB Service is running
2024-10-16T03:59:20Z INFO Hypervisor Starting watcher on dbService
2024-10-16T03:59:20Z INFO MySQL DB is not ready {"error": "dial tcp 127.0.0.1:3306: connect: connection refused"}
== DB == 2024-10-16 03:59:20+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 8.0.38-1.el9 started.
== DB == 2024-10-16 03:59:21+00:00 [Note] [Entrypoint]: Switching to dedicated user 'mysql'
== DB == 2024-10-16 03:59:21+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 8.0.38-1.el9 started.
== DB == '/var/lib/mysql/mysql.sock' -> '/var/run/mysqld/mysqld.sock'
2024-10-16T03:59:25Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:30Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:35Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:40Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:45Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:50Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:55Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T04:00:00Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
...
kubectl logs asmysql-test-bk2-mysql-0 lorry
2024-10-16T03:59:27Z INFO Initialize DB manager
2024-10-16T03:59:27Z INFO KB_WORKLOAD_TYPE ENV not set
2024-10-16T03:59:27Z INFO Volume-Protection succeed to init volume protection {"pod": "asmysql-test-bk2-mysql-0", "spec": {"highWatermark":"0","volumes":[]}}
2024-10-16T03:59:27Z INFO HTTPServer Starting HTTP Server
2024-10-16T03:59:27Z INFO HTTPServer API route path {"method": "POST", "path": ["/v1.0/switchover", "/v1.0/postprovision", "/v1.0/dataload", "/v1.0/getlag", "/v1.0/revokeuserrole", "/v1.0/preterminate", "/v1.0/deleteuser", "/v1.0/joinmember", "/v1.0/leavemember", "/v1.0/exec", "/v1.0/createuser", "/v1.0/grantuserrole", "/v1.0/volumeprotection", "/v1.0/datadump", "/v1.0/lockinstance", "/v1.0/rebuild", "/v1.0/unlockinstance", "/v1.0/checkrunning"]}
2024-10-16T03:59:27Z INFO HTTPServer API route path {"method": "GET", "path": ["/v1.0/query", "/v1.0/describeuser", "/v1.0/checkrole", "/v1.0/getrole", "/v1.0/listusers", "/v1.0/listsystemaccounts", "/v1.0/healthycheck"]}
2024-10-16T03:59:27Z INFO cronjobs env is not set {"env": "KB_CRON_JOBS"}
2024-10-16T03:59:33Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:33Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:34Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:35Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:36Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:37Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:38Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:39Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:40Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
2024-10-16T03:59:41Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"}
...
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.
Describe the bug
To Reproduce Steps to reproduce the behavior:
kbcli cluster restore asmysql-test-bk2 --backup backup-default-asmysql-test-20241016115559
➜ ~ kubectl get cluster asmysql-test-bk2 NAME CLUSTER-DEFINITION VERSION TERMINATION-POLICY STATUS AGE asmysql-test-bk2 mysql WipeOut Creating 9m57s ➜ ~ ➜ ~ kbcli cluster list-instances asmysql-test-bk2 NAME NAMESPACE CLUSTER COMPONENT STATUS ROLE ACCESSMODE AZ CPU(REQUEST/LIMIT) MEMORY(REQUEST/LIMIT) STORAGE NODE CREATED-TIME cn-shanghai-b 100m / 100m 512Mi / 512Mi data:20Gi 172.16.0.2/172.16.0.2 Oct 16,2024 11:58 UTC+0800 cn-shanghai-b 100m / 100m 512Mi / 512Mi data:20Gi 172.16.0.72/172.16.0.72 Oct 16,2024 11:58 UTC+0800
asmysql-test-bk2-mysql-0 default asmysql-test-bk2 mysql Running
asmysql-test-bk2-mysql-1 default asmysql-test-bk2 mysql Running
➜ ~
kubectl logs asmysql-test-bk2-mysql-0
Defaulted container "mysql" out of: mysql, lorry, config-manager, init-data (init), init-xtrabackup (init), init-syncer (init) 2024-10-16T03:59:26Z INFO Initialize DB manager 2024-10-16T03:59:26Z DEBUG KB_WORKLOAD_TYPE ENV not set 2024-10-16T03:59:26Z INFO HTTPServer Starting HTTP Server 2024-10-16T03:59:26Z INFO HTTPServer API route path {"method": "POST", "path": ["/v1.0/stop", "/v1.0/switchover", "/v1.0/rebuild", "/v1.0/start"]} 2024-10-16T03:59:26Z INFO HTTPServer API route path {"method": "GET", "path": ["/v1.0/datasync"]} 2024-10-16T03:59:26Z INFO HTTPServer http server {"listen address": "0.0.0.0", "port": 3601} 2024-10-16T03:59:26Z INFO HA HA starting 2024-10-16T03:59:26Z INFO pinger Waiting for dns resolution to be ready 2024-10-16T03:59:27Z INFO pinger dns resolution is ready {"dns": "asmysql-test-bk2-mysql-0.asmysql-test-bk2-mysql-headless.default.svc"} 2024-10-16T03:59:27Z INFO DCS-K8S pod selector: app.kubernetes.io/instance=asmysql-test-bk2,app.kubernetes.io/managed-by=kubeblocks,apps.kubeblocks.io/component-name=mysql 2024-10-16T03:59:27Z INFO DCS-K8S podlist: 2 2024-10-16T03:59:27Z INFO DCS-K8S Leader configmap is not found {"configmap": "asmysql-test-bk2-mysql-leader"} 2024-10-16T03:59:27Z DEBUG HA cluster info {"cluster": {"ClusterCompName":"asmysql-test-bk2-mysql","Namespace":"default","Replicas":2,"HaConfig":{"DeleteMembers":null},"Leader":null,"Members":[{"Index":"","Name":"asmysql-test-bk2-mysql-0","Role":"","PodIP":"172.16.0.54","DBPort":"3306","SyncerPort":"3601","UID":"c9a117df-b8b4-4a96-80f9-c4a86cc257f1","ComponentName":"mysql","UseIP":false},{"Index":"","Name":"asmysql-test-bk2-mysql-1","Role":"","PodIP":"172.16.0.77","DBPort":"3306","SyncerPort":"3601","UID":"b9b70059-11a3-4193-9dfc-aa7a9b847a0e","ComponentName":"mysql","UseIP":false}],"Switchover":null,"Extra":null,"Resource":{"metadata":{"name":"asmysql-test-bk2","namespace":"default","uid":"ab492447-8c97-4b24-93e0-68d264ee43d2","resourceVersion":"541645","generation":1,"creationTimestamp":"2024-10-16T03:57:50Z","labels":{"clusterdefinition.kubeblocks.io/name":"mysql","clusterversion.kubeblocks.io/name":""},"annotations":{"kubeblocks.io/ops-request":"[{\"name\":\"asmysql-test-bk2\",\"type\":\"Restore\"}]","kubeblocks.io/restore-from-backup":"{\"mysql\":{\"doReadyRestoreAfterClusterRunning\":\"false\",\"name\":\"backup-default-asmysql-test-20241016115559\",\"namespace\":\"default\",\"volumeRestorePolicy\":\"Parallel\"}}"},"finalizers":["cluster.kubeblocks.io/finalizer"],"managedFields":[{"manager":"kubeblocks","operation":"Update","apiVersion":"apps.kubeblocks.io/v1alpha1","time":"2024-10-16T03:57:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubeblocks.io/ops-request":{},"f:kubeblocks.io/restore-from-backup":{}},"f:finalizers":{".":{},"v:\"cluster.kubeblocks.io/finalizer\"":{}},"f:labels":{".":{},"f:clusterdefinition.kubeblocks.io/name":{},"f:clusterversion.kubeblocks.io/name":{}}},"f:spec":{".":{},"f:clusterDefinitionRef":{},"f:componentSpecs":{},"f:resources":{".":{},"f:cpu":{},"f:memory":{}},"f:storage":{".":{},"f:size":{}},"f:terminationPolicy":{},"f:topology":{}}}},{"manager":"kubeblocks","operation":"Update","apiVersion":"apps.kubeblocks.io/v1alpha1","time":"2024-10-16T03:58:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{".":{},"f:clusterDefGeneration":{},"f:components":{".":{},"f:mysql":{".":{},"f:phase":{},"f:podsReady":{}}},"f:conditions":{},"f:observedGeneration":{},"f:phase":{}}},"subresource":"status"}]},"spec":{"clusterDefinitionRef":"mysql","terminationPolicy":"WipeOut","componentSpecs":[{"name":"mysql","componentDef":"mysql-8.0","replicas":2,"resources":{"limits":{"cpu":"100m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeClaimTemplates":[{"name":"data","spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"20Gi"}}}}]}],"resources":{"cpu":"0","memory":"0"},"storage":{"size":"0"},"monitor":{}},"status":{"observedGeneration":1,"phase":"Creating","components":{"mysql":{"phase":"Creating","podsReady":false}},"clusterDefGeneration":2,"conditions":[{"type":"ProvisioningStarted","status":"True","observedGeneration":1,"lastTransitionTime":"2024-10-16T03:57:50Z","reason":"PreCheckSucceed","message":"The operator has started the provisioning of Cluster: asmysql-test-bk2"},{"type":"ApplyResources","status":"True","observedGeneration":1,"lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ApplyResourcesSucceed","message":"Successfully applied for resources"},{"type":"ReplicasReady","status":"False","lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ReplicasNotReady","message":"pods are not ready in Components: [mysql], refer to related component message in Cluster.status.components"},{"type":"Ready","status":"False","lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ComponentsNotReady","message":"pods are unavailable in Components: [mysql], refer to related component message in Cluster.status.components"}]}}}} 2024-10-16T03:59:27Z INFO Hypervisor Starting Hypervisor 2024-10-16T03:59:27Z INFO Hypervisor Start DB Service {"command": "/usr/bin/bash -c mv /var/lib/mysql/plugin/audit_log.so /usr/lib64/mysql/plugin/\nrm -rf /var/lib/mysql/plugin\nchown -R mysql:root /var/lib/mysql\nskip_slave_start=\"OFF\"\nif [ -f /var/lib/mysql/data/.restore_new_cluster ]; then\n skip_slave_start=\"ON\"\nfi\n/scripts/mysql-entrypoint.sh\n"} 2024-10-16T03:59:27Z INFO MySQL there is no leader member 2024-10-16T03:59:27Z INFO HA check if DB Service is running 2024-10-16T03:59:27Z INFO Hypervisor Starting watcher on dbService 2024-10-16T03:59:27Z INFO MySQL DB is not ready {"error": "dial tcp 127.0.0.1:3306: connect: connection refused"} == DB == 2024-10-16 03:59:28+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 8.0.38-1.el9 started. == DB == 2024-10-16 03:59:29+00:00 [Note] [Entrypoint]: Switching to dedicated user 'mysql' == DB == 2024-10-16 03:59:29+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 8.0.38-1.el9 started. == DB == '/var/lib/mysql/mysql.sock' -> '/var/run/mysqld/mysqld.sock' 2024-10-16T03:59:32Z INFO MySQL DB is not ready {"error": "dial tcp 127.0.0.1:3306: connect: connection refused"} 2024-10-16T03:59:38Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:43Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:48Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:53Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:58Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T04:00:03Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T04:00:08Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} ...
kubectl logs asmysql-test-bk2-mysql-1
Defaulted container "mysql" out of: mysql, lorry, config-manager, init-data (init), init-xtrabackup (init), init-syncer (init) 2024-10-16T03:59:19Z INFO Initialize DB manager 2024-10-16T03:59:19Z DEBUG KB_WORKLOAD_TYPE ENV not set 2024-10-16T03:59:19Z INFO HTTPServer Starting HTTP Server 2024-10-16T03:59:19Z INFO HTTPServer API route path {"method": "GET", "path": ["/v1.0/datasync"]} 2024-10-16T03:59:19Z INFO HTTPServer API route path {"method": "POST", "path": ["/v1.0/rebuild", "/v1.0/start", "/v1.0/stop", "/v1.0/switchover"]} 2024-10-16T03:59:19Z INFO HTTPServer http server {"listen address": "0.0.0.0", "port": 3601} 2024-10-16T03:59:19Z INFO HA HA starting 2024-10-16T03:59:19Z INFO pinger Waiting for dns resolution to be ready 2024-10-16T03:59:19Z INFO pinger dns resolution is ready {"dns": "asmysql-test-bk2-mysql-1.asmysql-test-bk2-mysql-headless.default.svc"} 2024-10-16T03:59:20Z INFO DCS-K8S pod selector: app.kubernetes.io/instance=asmysql-test-bk2,app.kubernetes.io/managed-by=kubeblocks,apps.kubeblocks.io/component-name=mysql 2024-10-16T03:59:20Z INFO DCS-K8S podlist: 2 2024-10-16T03:59:20Z INFO DCS-K8S Leader configmap is not found {"configmap": "asmysql-test-bk2-mysql-leader"} 2024-10-16T03:59:20Z DEBUG HA cluster info {"cluster": {"ClusterCompName":"asmysql-test-bk2-mysql","Namespace":"default","Replicas":2,"HaConfig":{"DeleteMembers":null},"Leader":null,"Members":[{"Index":"","Name":"asmysql-test-bk2-mysql-0","Role":"","PodIP":"172.16.0.54","DBPort":"3306","SyncerPort":"3601","UID":"c9a117df-b8b4-4a96-80f9-c4a86cc257f1","ComponentName":"mysql","UseIP":false},{"Index":"","Name":"asmysql-test-bk2-mysql-1","Role":"","PodIP":"172.16.0.77","DBPort":"3306","SyncerPort":"3601","UID":"b9b70059-11a3-4193-9dfc-aa7a9b847a0e","ComponentName":"mysql","UseIP":false}],"Switchover":null,"Extra":null,"Resource":{"metadata":{"name":"asmysql-test-bk2","namespace":"default","uid":"ab492447-8c97-4b24-93e0-68d264ee43d2","resourceVersion":"541645","generation":1,"creationTimestamp":"2024-10-16T03:57:50Z","labels":{"clusterdefinition.kubeblocks.io/name":"mysql","clusterversion.kubeblocks.io/name":""},"annotations":{"kubeblocks.io/ops-request":"[{\"name\":\"asmysql-test-bk2\",\"type\":\"Restore\"}]","kubeblocks.io/restore-from-backup":"{\"mysql\":{\"doReadyRestoreAfterClusterRunning\":\"false\",\"name\":\"backup-default-asmysql-test-20241016115559\",\"namespace\":\"default\",\"volumeRestorePolicy\":\"Parallel\"}}"},"finalizers":["cluster.kubeblocks.io/finalizer"],"managedFields":[{"manager":"kubeblocks","operation":"Update","apiVersion":"apps.kubeblocks.io/v1alpha1","time":"2024-10-16T03:57:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubeblocks.io/ops-request":{},"f:kubeblocks.io/restore-from-backup":{}},"f:finalizers":{".":{},"v:\"cluster.kubeblocks.io/finalizer\"":{}},"f:labels":{".":{},"f:clusterdefinition.kubeblocks.io/name":{},"f:clusterversion.kubeblocks.io/name":{}}},"f:spec":{".":{},"f:clusterDefinitionRef":{},"f:componentSpecs":{},"f:resources":{".":{},"f:cpu":{},"f:memory":{}},"f:storage":{".":{},"f:size":{}},"f:terminationPolicy":{},"f:topology":{}}}},{"manager":"kubeblocks","operation":"Update","apiVersion":"apps.kubeblocks.io/v1alpha1","time":"2024-10-16T03:58:53Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{".":{},"f:clusterDefGeneration":{},"f:components":{".":{},"f:mysql":{".":{},"f:phase":{},"f:podsReady":{}}},"f:conditions":{},"f:observedGeneration":{},"f:phase":{}}},"subresource":"status"}]},"spec":{"clusterDefinitionRef":"mysql","terminationPolicy":"WipeOut","componentSpecs":[{"name":"mysql","componentDef":"mysql-8.0","replicas":2,"resources":{"limits":{"cpu":"100m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"512Mi"}},"volumeClaimTemplates":[{"name":"data","spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"20Gi"}}}}]}],"resources":{"cpu":"0","memory":"0"},"storage":{"size":"0"},"monitor":{}},"status":{"observedGeneration":1,"phase":"Creating","components":{"mysql":{"phase":"Creating","podsReady":false}},"clusterDefGeneration":2,"conditions":[{"type":"ProvisioningStarted","status":"True","observedGeneration":1,"lastTransitionTime":"2024-10-16T03:57:50Z","reason":"PreCheckSucceed","message":"The operator has started the provisioning of Cluster: asmysql-test-bk2"},{"type":"ApplyResources","status":"True","observedGeneration":1,"lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ApplyResourcesSucceed","message":"Successfully applied for resources"},{"type":"ReplicasReady","status":"False","lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ReplicasNotReady","message":"pods are not ready in Components: [mysql], refer to related component message in Cluster.status.components"},{"type":"Ready","status":"False","lastTransitionTime":"2024-10-16T03:57:50Z","reason":"ComponentsNotReady","message":"pods are unavailable in Components: [mysql], refer to related component message in Cluster.status.components"}]}}}} 2024-10-16T03:59:20Z INFO Hypervisor Starting Hypervisor 2024-10-16T03:59:20Z INFO Hypervisor Start DB Service {"command": "/usr/bin/bash -c mv /var/lib/mysql/plugin/audit_log.so /usr/lib64/mysql/plugin/\nrm -rf /var/lib/mysql/plugin\nchown -R mysql:root /var/lib/mysql\nskip_slave_start=\"OFF\"\nif [ -f /var/lib/mysql/data/.restore_new_cluster ]; then\n skip_slave_start=\"ON\"\nfi\n/scripts/mysql-entrypoint.sh\n"} 2024-10-16T03:59:20Z INFO MySQL there is no leader member 2024-10-16T03:59:20Z INFO HA check if DB Service is running 2024-10-16T03:59:20Z INFO Hypervisor Starting watcher on dbService 2024-10-16T03:59:20Z INFO MySQL DB is not ready {"error": "dial tcp 127.0.0.1:3306: connect: connection refused"} == DB == 2024-10-16 03:59:20+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 8.0.38-1.el9 started. == DB == 2024-10-16 03:59:21+00:00 [Note] [Entrypoint]: Switching to dedicated user 'mysql' == DB == 2024-10-16 03:59:21+00:00 [Note] [Entrypoint]: Entrypoint script for MySQL Server 8.0.38-1.el9 started. == DB == '/var/lib/mysql/mysql.sock' -> '/var/run/mysqld/mysqld.sock' 2024-10-16T03:59:25Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:30Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:35Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:40Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:45Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:50Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:55Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T04:00:00Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} ...
kubectl logs asmysql-test-bk2-mysql-0 lorry 2024-10-16T03:59:27Z INFO Initialize DB manager 2024-10-16T03:59:27Z INFO KB_WORKLOAD_TYPE ENV not set 2024-10-16T03:59:27Z INFO Volume-Protection succeed to init volume protection {"pod": "asmysql-test-bk2-mysql-0", "spec": {"highWatermark":"0","volumes":[]}} 2024-10-16T03:59:27Z INFO HTTPServer Starting HTTP Server 2024-10-16T03:59:27Z INFO HTTPServer API route path {"method": "POST", "path": ["/v1.0/switchover", "/v1.0/postprovision", "/v1.0/dataload", "/v1.0/getlag", "/v1.0/revokeuserrole", "/v1.0/preterminate", "/v1.0/deleteuser", "/v1.0/joinmember", "/v1.0/leavemember", "/v1.0/exec", "/v1.0/createuser", "/v1.0/grantuserrole", "/v1.0/volumeprotection", "/v1.0/datadump", "/v1.0/lockinstance", "/v1.0/rebuild", "/v1.0/unlockinstance", "/v1.0/checkrunning"]} 2024-10-16T03:59:27Z INFO HTTPServer API route path {"method": "GET", "path": ["/v1.0/query", "/v1.0/describeuser", "/v1.0/checkrole", "/v1.0/getrole", "/v1.0/listusers", "/v1.0/listsystemaccounts", "/v1.0/healthycheck"]} 2024-10-16T03:59:27Z INFO cronjobs env is not set {"env": "KB_CRON_JOBS"} 2024-10-16T03:59:33Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:33Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:34Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:35Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:36Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:37Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:38Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:39Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:40Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} 2024-10-16T03:59:41Z INFO MySQL DB is not ready {"error": "Error 1045 (28000): Access denied for user 'root'@'127.0.0.1' (using password: YES)"} ...