Y-Rookie
commented
1 year ago ➜ ~ k logs redis-test-redis-1
Defaulted container "redis" out of: redis, metrics, kb-checkrole
+ echo include /etc/conf/redis.conf
+ echo replica-announce-ip redis-test-redis-1.redis-test-redis-headless.default.svc
+ [ -f /data/users.acl ]
+ touch /data/users.acl
+ [ ! -z 6vkl4n7c ]
+ echo masteruser kbreplicator
+ echo masterauth 6vkl4n7c
+ echo user kbreplicator on +psync +replconf +ping >6vkl4n7c
+ [ ! -z 6vkl4n7c ]
+ echo user kbreplicator-sentinel on allchannels +multi +slaveof +ping +exec +subscribe +config|rewrite +role +publish +info +client|setname +client|kill +script|kill >6vkl4n7c
+ [ ! -z 6vkl4n7c ]
+ echo protected-mode yes
+ echo user default on allcommands allkeys >6vkl4n7c
+ echo aclfile /data/users.acl
+ start_redis_server
+ exec redis-server /etc/redis/redis.conf --loadmodule /opt/redis-stack/lib/redisearch.so --loadmodule /opt/redis-stack/lib/redisgraph.so --loadmodule /opt/redis-stack/lib/redistimeseries.so --loadmodule /opt/redis-stack/lib/rejson.so --loadmodule /opt/redis-stack/lib/redisbloom.so
+ create_replication
+ [ ! -z 6vkl4n7c ]
+ retry redis-cli -h 127.0.0.1 -p 6379 -a 6vkl4n7c ping
+ local max_attempts=20
+ local attempt=1
+ redis-cli -h 127.0.0.1 -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at 127.0.0.1:6379: Connection refused
+ [ 1 -eq 20 ]
+ echo Command 'redis-cli -h 127.0.0.1 -p 6379 -a 6vkl4n7c ping' failed. Attempt 1 of 20. Retrying in 5 seconds...
+ attempt=2
+ sleep 3
Command 'redis-cli -h 127.0.0.1 -p 6379 -a 6vkl4n7c ping' failed. Attempt 1 of 20. Retrying in 5 seconds...
+ redis-cli -h 127.0.0.1 -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
PONG
+ [ 2 -eq 20 ]
+ attempt=1
+ max_attempts=20
+ [ 1 -le 20 ]
+ cat /kb-podinfo/primary-pod
+ [ -z redis-test-redis-0 ]
+ cat /kb-podinfo/primary-pod
+ primary=redis-test-redis-0
+ echo DownwardAPI get primary=redis-test-redis-0
+ echo KB_POD_NAME=redis-test-redis-1
+ [ -z redis-test-redis-0 ]
+ [ redis-test-redis-0 = redis-test-redis-1 ]
+ primary_fqdn=redis-test-redis-0.redis-test-redis-headless.default.svc
+ echo primary_fqdn=redis-test-redis-0.redis-test-redis-headless.default.svc
+ [ ! -z 6vkl4n7c ]
+ retry redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
+ local max_attempts=20
+ local attempt=1
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 1 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 1 of 20. Retrying in 5 seconds...
+ attempt=2
+ sleep 3
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 1 of 20. Retrying in 5 seconds...
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 2 of 20. Retrying in 5 seconds...
+ [ 2 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 2 of 20. Retrying in 5 seconds...
+ attempt=3
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 3 of 20. Retrying in 5 seconds...
+ [ 3 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 3 of 20. Retrying in 5 seconds...
+ attempt=4
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 4 of 20. Retrying in 5 seconds...
+ [ 4 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 4 of 20. Retrying in 5 seconds...
+ attempt=5
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 5 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 5 of 20. Retrying in 5 seconds...
+ attempt=6
+ sleep 3
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 5 of 20. Retrying in 5 seconds...
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 6 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 6 of 20. Retrying in 5 seconds...
+ attempt=7
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 6 of 20. Retrying in 5 seconds...
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 7 of 20. Retrying in 5 seconds...
+ [ 7 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 7 of 20. Retrying in 5 seconds...
+ attempt=8
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 8 of 20. Retrying in 5 seconds...
+ [ 8 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 8 of 20. Retrying in 5 seconds...
+ attempt=9
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 9 of 20. Retrying in 5 seconds...
+ [ 9 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 9 of 20. Retrying in 5 seconds...
+ attempt=10
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 10 of 20. Retrying in 5 seconds...
+ [ 10 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 10 of 20. Retrying in 5 seconds...
+ attempt=11
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 11 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 11 of 20. Retrying in 5 seconds...
+ attempt=12
+ sleep 3
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 11 of 20. Retrying in 5 seconds...
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 12 of 20. Retrying in 5 seconds...
+ [ 12 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 12 of 20. Retrying in 5 seconds...
+ attempt=13
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 13 of 20. Retrying in 5 seconds...
+ [ 13 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 13 of 20. Retrying in 5 seconds...
+ attempt=14
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 14 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 14 of 20. Retrying in 5 seconds...
+ attempt=15
+ sleep 3
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 14 of 20. Retrying in 5 seconds...
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 15 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 15 of 20. Retrying in 5 seconds...
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 15 of 20. Retrying in 5 seconds...
+ attempt=16
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 16 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 16 of 20. Retrying in 5 seconds...
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 16 of 20. Retrying in 5 seconds...
+ attempt=17
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 17 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 17 of 20. Retrying in 5 seconds...
+ attempt=18
+ sleep 3
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 17 of 20. Retrying in 5 seconds...
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 18 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 18 of 20. Retrying in 5 seconds...
+ attempt=19
+ sleep 3
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 18 of 20. Retrying in 5 seconds...
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
+ [ 19 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 19 of 20. Retrying in 5 seconds...
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed. Attempt 19 of 20. Retrying in 5 seconds...
+ attempt=20
+ sleep 3
+ redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at redis-test-redis-0.redis-test-redis-headless.default.svc:6379: Name or service not known
Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed after 20 attempts. Exiting...
+ [ 20 -eq 20 ]
+ [ 20 -eq 20 ]
+ echo Command 'redis-cli -h redis-test-redis-0.redis-test-redis-headless.default.svc -p 6379 -a 6vkl4n7c ping' failed after 20 attempts. Exiting...
+ exit 1
JashBook
Describe the bug redis cluster status sometimes always creating and both pod labels are primary. the number of primary pod is not equal to 1, primary pods: [redis-test-redis-1 redis-test-redis-0],
To Reproduce Steps to reproduce the behavior:
kubectl get sts NAME READY AGE redis-test-redis 2/2 65m redis-test-redis-sentinel 3/3 65m
kubectl describe cluster redis-test Name: redis-test Namespace: default Labels: clusterdefinition.kubeblocks.io/name=redis clusterversion.kubeblocks.io/name=redis-7.0.6 Annotations: kubeblocks.io/reconcile: 2023-08-01T07:54:49.227401226Z API Version: apps.kubeblocks.io/v1alpha1 Kind: Cluster Metadata: Creation Timestamp: 2023-08-01T06:54:01Z Finalizers: cluster.kubeblocks.io/finalizer Generation: 1 Resource Version: 29236423 UID: 131641be-3989-4ea0-80e7-822bc33e2520 Spec: Affinity: Node Labels: Pod Anti Affinity: Preferred Tenancy: SharedNode Topology Keys: Cluster Definition Ref: redis Cluster Version Ref: redis-7.0.6 Component Specs: Component Def Ref: redis Monitor: false Name: redis No Create PDB: false Replicas: 2 Resources: Limits: Cpu: 100m Memory: 512Mi Requests: Cpu: 100m Memory: 512Mi Service Account Name: kb-redis-test Switch Policy: Type: Noop Volume Claim Templates: Name: data Spec: Access Modes: ReadWriteOnce Resources: Requests: Storage: 1Gi Component Def Ref: redis-sentinel Monitor: false Name: redis-sentinel No Create PDB: false Replicas: 3 Resources: Limits: Cpu: 100m Memory: 512Mi Requests: Cpu: 100m Memory: 512Mi Service Account Name: kb-redis-test Volume Claim Templates: Name: data Spec: Access Modes: ReadWriteOnce Resources: Requests: Storage: 1Gi Termination Policy: WipeOut Tolerations: Status: Cluster Def Generation: 3 Components: Redis: Phase: Creating Pods Ready: false Replication Set Status: Primary: Pod: redis-test-redis-0 Secondaries: Pod: redis-test-redis-1 Redis - Sentinel: Phase: Creating Pods Ready: false Conditions: Last Transition Time: 2023-08-01T06:54:01Z Message: The operator has started the provisioning of Cluster: redis-test Observed Generation: 1 Reason: PreCheckSucceed Status: True Type: ProvisioningStarted Last Transition Time: 2023-08-01T06:54:27Z Message: the number of primary pod is not equal to 1, primary pods: [redis-test-redis-1 redis-test-redis-0], emptyRole pods: [] Reason: ApplyResourcesFailed Status: False Type: ApplyResources Last Transition Time: 2023-08-01T06:54:02Z Message: pods are not ready in Components: [redis redis-sentinel], refer to related component message in Cluster.status.components Reason: ReplicasNotReady Status: False Type: ReplicasReady Last Transition Time: 2023-08-01T06:54:02Z Message: pods are unavailable in Components: [redis redis-sentinel], refer to related component message in Cluster.status.components Reason: ComponentsNotReady Status: False Type: Ready Observed Generation: 1 Phase: Creating Events: Type Reason Age From Message
Warning ApplyResourcesFailed 7m40s (x63 over 65m) cluster-controller the number of primary pod is not equal to 1, primary pods: [redis-test-redis-1 redis-test-redis-0], emptyRole pods: [] Warning Unhealthy 7m7s (x8 over 7m42s) event-controller Pod redis-test-redis-sentinel-2: Readiness probe errored: command "sh -c /scripts/redis-sentinel-ping.sh 1" timed out Warning Unhealthy 7m7s (x2 over 7m37s) event-controller Pod redis-test-redis-sentinel-0: Liveness probe errored: command "sh -c /scripts/redis-sentinel-ping.sh 5" timed out Warning Unhealthy 7m5s (x9 over 7m45s) event-controller Pod redis-test-redis-sentinel-0: Readiness probe errored: command "sh -c /scripts/redis-sentinel-ping.sh 1" timed out Warning Unhealthy 7m4s (x3 over 7m34s) event-controller Pod redis-test-redis-sentinel-2: Liveness probe errored: command "sh -c /scripts/redis-sentinel-ping.sh 5" timed out Warning Unhealthy 7m2s (x3 over 7m42s) event-controller Pod redis-test-redis-sentinel-1: Readiness probe errored: command "sh -c /scripts/redis-sentinel-ping.sh 1" timed out Warning ApplyResourcesFailed 5m57s (x116 over 65m) cluster-controller the number of primary pod is not equal to 1, primary pods: [redis-test-redis-0 redis-test-redis-1], emptyRole pods: []