Open gyliu513 opened 4 years ago
I saw the same issue in my env w/ this error:
MountVolume.SetUp failed for volume "ca" : secret "multicloud-ca-cert" not found
Also, I didn't see below pods in my env:
cert-manager-e61c1-6cf697d5df-rhhcc
cert-manager-webhook-03d4d-cainjector-8d76f6646-t5crm
cert-manager-webhook-74bdc8455d-jn6qf
I have a similar problem. I installed "Advanced Cluster Management for Kubernetes" Operator via the OperatorHub. Created a simple MultiClusterHub and got stuck at this point. I followed the official red hat documentation.
Using Openshft version 4.4.8 installed on vSphere UPI.
[root@helpernode ~]# oc get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
master0.openshiftcluster.lab.local Ready master 4d23h v1.17.1+3f6f40d 10.150.176.207 10.150.176.207 Red Hat Enterprise Linux CoreOS 44.81.202006080130-0 (Ootpa) 4.18.0-147.8.1.el8_1.x86_64 cri-o://1.17.4-14.dev.rhaos4.4.gitb93af5d.el8
master1.openshiftcluster.lab.local Ready master 5d1h v1.17.1+3f6f40d 10.150.176.208 10.150.176.208 Red Hat Enterprise Linux CoreOS 44.81.202006080130-0 (Ootpa) 4.18.0-147.8.1.el8_1.x86_64 cri-o://1.17.4-14.dev.rhaos4.4.gitb93af5d.el8
master2.openshiftcluster.lab.local Ready master 10d v1.17.1+3f6f40d 10.150.176.209 10.150.176.209 Red Hat Enterprise Linux CoreOS 44.81.202006080130-0 (Ootpa) 4.18.0-147.8.1.el8_1.x86_64 cri-o://1.17.4-14.dev.rhaos4.4.gitb93af5d.el8
worker0.lab.local Ready worker 5d1h v1.17.1+3f6f40d 10.150.176.210 10.150.176.210 Red Hat Enterprise Linux CoreOS 44.81.202006080130-0 (Ootpa) 4.18.0-147.8.1.el8_1.x86_64 cri-o://1.17.4-14.dev.rhaos4.4.gitb93af5d.el8
worker1.openshiftcluster.lab.local Ready worker 10d v1.17.1+3f6f40d 10.150.176.211 10.150.176.211 Red Hat Enterprise Linux CoreOS 44.81.202006080130-0 (Ootpa) 4.18.0-147.8.1.el8_1.x86_64 cri-o://1.17.4-14.dev.rhaos4.4.gitb93af5d.el8
worker2.lab.local Ready worker 5d1h v1.17.1+3f6f40d 10.150.176.212 10.150.176.212 Red Hat Enterprise Linux CoreOS 44.81.202006080130-0 (Ootpa) 4.18.0-147.8.1.el8_1.x86_64 cri-o://1.17.4-14.dev.rhaos4.4.gitb93af5d.el8
[root@helpernode ~]# oc version
Client Version: version.Info{Major:"4", Minor:"1+", GitVersion:"v4.1.18-201909201915+72d1bea-dirty", GitCommit:"72d1bea", GitTreeState:"dirty", BuildDate:"2019-09-21T02:11:40Z", GoVersion:"go1.11.13", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"17+", GitVersion:"v1.17.1+3f6f40d", GitCommit:"3f6f40d", GitTreeState:"clean", BuildDate:"2020-06-08T07:13:25Z", GoVersion:"go1.13.4", Compiler:"gc", Platform:"linux/amd64"}
All resources from namespace open-cluster-management.
[root@helpernode ~]# oc get all -n open-cluster-management
NAME READY STATUS RESTARTS AGE
pod/application-chart-5a30b-applicationui-75b7656f8b-tpznc 1/1 Running 0 2d5h
pod/configmap-watcher-708df-766b45dc94-zfzv8 1/1 Running 0 2d5h
pod/etcd-operator-5f96987979-bkv7m 3/3 Running 0 2d5h
pod/hive-operator-f87cb8795-9hbqq 1/1 Running 0 2d5h
pod/mcm-apiserver-564dfd455-44pqg 0/1 ContainerCreating 0 32h
pod/mcm-controller-5f875c7695-zpxxn 1/1 Running 0 2d5h
pod/mcm-webhook-5978645464-nwr4h 1/1 Running 0 2d5h
pod/multicluster-operators-application-7c8dbb89c5-8x65f 4/4 Running 0 2d5h
pod/multicluster-operators-hub-subscription-96c947f4-zlw8w 1/1 Running 0 2d5h
pod/multicluster-operators-standalone-subscription-657746c9d5-8g9vw 1/1 Running 4 2d5h
pod/multiclusterhub-operator-6b747fc95-lnxlw 1/1 Running 0 32h
pod/multiclusterhub-repo-794c964dcf-l8hfl 1/1 Running 0 2d5h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/application-chart-5a30b-applicationui ClusterIP 172.30.59.166 <none> 3001/TCP 2d5h
service/etcd-restore-operator ClusterIP 172.30.3.165 <none> 19999/TCP 2d5h
service/mcm-apiserver ClusterIP 172.30.61.5 <none> 443/TCP 2d5h
service/mcm-webhook ClusterIP 172.30.29.252 <none> 443/TCP 2d5h
service/multicluster-operators-application-metrics ClusterIP 172.30.35.242 <none> 8386/TCP,8689/TCP 2d5h
service/multicluster-operators-channel-metrics ClusterIP 172.30.28.246 <none> 8384/TCP,8687/TCP 2d5h
service/multicluster-operators-deployable-metrics ClusterIP 172.30.125.23 <none> 8385/TCP,8688/TCP 2d5h
service/multicluster-operators-hub-subscription-metrics ClusterIP 172.30.55.92 <none> 8383/TCP,8686/TCP 2d5h
service/multicluster-operators-placementrule-metrics ClusterIP 172.30.146.248 <none> 8383/TCP,8686/TCP 2d5h
service/multicluster-operators-standalone-subscription-metrics ClusterIP 172.30.84.10 <none> 8383/TCP,8686/TCP 2d5h
service/multiclusterhub-operator-metrics ClusterIP 172.30.81.143 <none> 8383/TCP,8686/TCP 32h
service/multiclusterhub-operator-webhook ClusterIP 172.30.220.212 <none> 443/TCP 32h
service/multiclusterhub-repo ClusterIP 172.30.40.158 <none> 3000/TCP 2d5h
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/application-chart-5a30b-applicationui 1/1 1 1 2d5h
deployment.apps/configmap-watcher-708df 1/1 1 1 2d5h
deployment.apps/etcd-operator 1/1 1 1 2d5h
deployment.apps/hive-operator 1/1 1 1 2d5h
deployment.apps/mcm-apiserver 0/1 1 0 2d5h
deployment.apps/mcm-controller 1/1 1 1 2d5h
deployment.apps/mcm-webhook 1/1 1 1 2d5h
deployment.apps/multicluster-operators-application 1/1 1 1 2d5h
deployment.apps/multicluster-operators-hub-subscription 1/1 1 1 2d5h
deployment.apps/multicluster-operators-standalone-subscription 1/1 1 1 2d5h
deployment.apps/multiclusterhub-operator 1/1 1 1 2d5h
deployment.apps/multiclusterhub-repo 1/1 1 1 2d5h
NAME DESIRED CURRENT READY AGE
replicaset.apps/application-chart-5a30b-applicationui-75b7656f8b 1 1 1 2d5h
replicaset.apps/configmap-watcher-708df-766b45dc94 1 1 1 2d5h
replicaset.apps/etcd-operator-5f96987979 1 1 1 2d5h
replicaset.apps/hive-operator-f87cb8795 1 1 1 2d5h
replicaset.apps/mcm-apiserver-564dfd455 1 1 0 2d5h
replicaset.apps/mcm-controller-5f875c7695 1 1 1 2d5h
replicaset.apps/mcm-webhook-5978645464 1 1 1 2d5h
replicaset.apps/multicluster-operators-application-7c8dbb89c5 1 1 1 2d5h
replicaset.apps/multicluster-operators-hub-subscription-96c947f4 1 1 1 2d5h
replicaset.apps/multicluster-operators-standalone-subscription-657746c9d5 1 1 1 2d5h
replicaset.apps/multiclusterhub-operator-6b747fc95 1 1 1 32h
replicaset.apps/multiclusterhub-repo-794c964dcf 1 1 1 2d5h
Failing Pod Description:
[root@helpernode ~]# oc describe pod mcm-apiserver-564dfd455-44pqg
Name: mcm-apiserver-564dfd455-44pqg
Namespace: open-cluster-management
Priority: 0
PriorityClassName: <none>
Node: worker2.lab.local/10.150.176.212
Start Time: Sat, 27 Jun 2020 07:13:02 +0100
Labels: app=mcm-apiserver
pod-template-hash=564dfd455
Annotations: openshift.io/scc: anyuid
Status: Pending
IP:
Controlled By: ReplicaSet/mcm-apiserver-564dfd455
Containers:
mcm-apiserver:
Container ID:
Image: registry.redhat.io/rhacm1-tech-preview/multicloud-manager-rhel8@sha256:a8fb58443c3177e5ac78933a35198a75b7746396d68f477d484b4e0c4bc7d295
Image ID:
Port: <none>
Host Port: <none>
Args:
/mcm-apiserver
--mongo-database=mcm
--enable-admission-plugins=HCMUserIdentity,KlusterletCA,NamespaceLifecycle
--secure-port=6443
--tls-cert-file=/var/run/apiserver/tls.crt
--tls-private-key-file=/var/run/apiserver/tls.key
--klusterlet-cafile=/var/run/klusterlet/ca.crt
--klusterlet-certfile=/var/run/klusterlet/tls.crt
--klusterlet-keyfile=/var/run/klusterlet/tls.key
--http2-max-streams-per-connection=1000
--etcd-servers=http://etcd-cluster.open-cluster-management.svc.cluster.local:2379
--mongo-host=multicluster-mongodb
--mongo-replicaset=rs0
State: Waiting
Reason: ContainerCreating
Ready: False
Restart Count: 0
Limits:
memory: 2Gi
Requests:
cpu: 200m
memory: 256Mi
Liveness: http-get https://:6443/healthz delay=2s timeout=1s period=10s #success=1 #failure=3
Readiness: http-get https://:6443/healthz delay=2s timeout=1s period=10s #success=1 #failure=3
Environment:
MONGO_USERNAME: <set to the key 'user' in secret 'mongodb-admin'> Optional: false
MONGO_PASSWORD: <set to the key 'password' in secret 'mongodb-admin'> Optional: false
MONGO_SSLCA: /certs/mongodb-ca/tls.crt
MONGO_SSLCERT: /certs/mongodb-client/tls.crt
MONGO_SSLKEY: /certs/mongodb-client/tls.key
Mounts:
/certs/mongodb-ca from mongodb-ca-cert (rw)
/certs/mongodb-client from mongodb-client-cert (rw)
/var/run/apiserver from apiserver-certs (rw)
/var/run/klusterlet from klusterlet-certs (rw)
/var/run/secrets/kubernetes.io/serviceaccount from hub-sa-token-6xwh2 (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
apiserver-certs:
Type: Secret (a volume populated by a Secret)
SecretName: mcm-apiserver-self-signed-secrets
Optional: false
klusterlet-certs:
Type: Secret (a volume populated by a Secret)
SecretName: mcm-klusterlet-self-signed-secrets
Optional: false
mongodb-ca-cert:
Type: Secret (a volume populated by a Secret)
SecretName: multicloud-ca-cert
Optional: false
mongodb-client-cert:
Type: Secret (a volume populated by a Secret)
SecretName: multicluster-mongodb-client-cert
Optional: false
hub-sa-token-6xwh2:
Type: Secret (a volume populated by a Secret)
SecretName: hub-sa-token-6xwh2
Optional: false
QoS Class: Burstable
Node-Selectors: <none>
Tolerations: node.kubernetes.io/memory-pressure:NoSchedule
node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedMount 176m (x136 over 33h) kubelet, worker2.lab.local Unable to attach or mount volumes: unmounted volumes=[mongodb-ca-cert mongodb-client-cert], unattached volumes=[klusterlet-certs mongodb-ca-cert mongodb-client-cert hub-sa-token-6xwh2 apiserver-certs]: timed out waiting for the condition
Warning FailedMount 152m (x148 over 33h) kubelet, worker2.lab.local Unable to attach or mount volumes: unmounted volumes=[mongodb-ca-cert mongodb-client-cert], unattached volumes=[hub-sa-token-6xwh2 apiserver-certs klusterlet-certs mongodb-ca-cert mongodb-client-cert]: timed out waiting for the condition
Warning FailedMount 90m (x134 over 33h) kubelet, worker2.lab.local Unable to attach or mount volumes: unmounted volumes=[mongodb-client-cert mongodb-ca-cert], unattached volumes=[mongodb-client-cert hub-sa-token-6xwh2 apiserver-certs klusterlet-certs mongodb-ca-cert]: timed out waiting for the condition
Warning FailedMount 66m (x967 over 33h) kubelet, worker2.lab.local MountVolume.SetUp failed for volume "mongodb-client-cert" : secret "multicluster-mongodb-client-cert" not found
Warning FailedMount 61m (x286 over 33h) kubelet, worker2.lab.local Unable to attach or mount volumes: unmounted volumes=[mongodb-ca-cert mongodb-client-cert], unattached volumes=[apiserver-certs klusterlet-certs mongodb-ca-cert mongodb-client-cert hub-sa-token-6xwh2]: timed out waiting for the condition
Warning FailedMount 56m (x145 over 33h) kubelet, worker2.lab.local Unable to attach or mount volumes: unmounted volumes=[mongodb-ca-cert mongodb-client-cert], unattached volumes=[mongodb-ca-cert mongodb-client-cert hub-sa-token-6xwh2 apiserver-certs klusterlet-certs]: timed out waiting for the condition
Warning FailedMount 52m (x974 over 33h) kubelet, worker2.lab.local MountVolume.SetUp failed for volume "mongodb-ca-cert" : secret "multicloud-ca-cert" not found
multiclusterhub YAML:
apiVersion: operators.open-cluster-management.io/v1beta1
kind: MultiClusterHub
metadata:
creationTimestamp: '2020-06-26T09:47:41Z'
finalizers:
- finalizer.operators.open-cluster-management.io
generation: 3
name: multiclusterhub
namespace: open-cluster-management
resourceVersion: '105195441'
selfLink: >-
/apis/operators.open-cluster-management.io/v1beta1/namespaces/open-cluster-management/multiclusterhubs/multiclusterhub
uid: 0c8894ef-476d-41e1-8051-d6ed50fea16e
spec:
cloudPakCompatibility: false
etcd:
storage: 1Gi
storageClass: thin
failover: false
hive:
backup:
velero: {}
failedProvisionConfig: {}
imagePullSecret: myclustersecret
ipv6: false
mongo:
storage: 5Gi
storageClass: thin
overrides: {}
status:
currentVersion: 1.0.1
deployments:
- name: hive-operator
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:22:04Z'
lastUpdateTime: '2020-06-26T09:22:04Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T09:21:12Z'
lastUpdateTime: '2020-06-26T09:22:04Z'
message: ReplicaSet "hive-operator-f87cb8795" has successfully progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 13
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: mcm-webhook
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:50:08Z'
lastUpdateTime: '2020-06-26T09:50:08Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T09:47:45Z'
lastUpdateTime: '2020-06-26T09:50:08Z'
message: ReplicaSet "mcm-webhook-5978645464" has successfully progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: multicluster-operators-hub-subscription
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:23:18Z'
lastUpdateTime: '2020-06-26T09:23:18Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T09:21:13Z'
lastUpdateTime: '2020-06-26T09:23:18Z'
message: >-
ReplicaSet "multicluster-operators-hub-subscription-96c947f4" has
successfully progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 13
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: multiclusterhub-repo
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:49:10Z'
lastUpdateTime: '2020-06-26T09:49:10Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T09:47:42Z'
lastUpdateTime: '2020-06-26T09:49:10Z'
message: >-
ReplicaSet "multiclusterhub-repo-794c964dcf" has successfully
progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: application-chart-5a30b-applicationui
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T10:04:27Z'
lastUpdateTime: '2020-06-26T10:04:27Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T10:04:27Z'
lastUpdateTime: '2020-06-26T10:04:27Z'
message: >-
ReplicaSet "application-chart-5a30b-applicationui-75b7656f8b" has
successfully progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: mcm-controller
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:50:09Z'
lastUpdateTime: '2020-06-26T09:50:09Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T09:47:45Z'
lastUpdateTime: '2020-06-26T09:50:09Z'
message: >-
ReplicaSet "mcm-controller-5f875c7695" has successfully
progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: etcd-operator
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:21:16Z'
lastUpdateTime: '2020-06-26T09:21:16Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T09:21:10Z'
lastUpdateTime: '2020-06-26T09:21:16Z'
message: ReplicaSet "etcd-operator-5f96987979" has successfully progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 2
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: multicluster-operators-standalone-subscription
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:21:12Z'
lastUpdateTime: '2020-06-26T09:22:00Z'
message: >-
ReplicaSet
"multicluster-operators-standalone-subscription-657746c9d5" has
successfully progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
- lastTransitionTime: '2020-06-28T07:53:45Z'
lastUpdateTime: '2020-06-28T07:53:45Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
observedGeneration: 13
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: mcm-apiserver
status:
conditions:
- lastTransitionTime: '2020-06-26T09:47:44Z'
lastUpdateTime: '2020-06-26T09:47:44Z'
message: Deployment does not have minimum availability.
reason: MinimumReplicasUnavailable
status: 'False'
type: Available
- lastTransitionTime: '2020-06-27T06:23:03Z'
lastUpdateTime: '2020-06-27T06:23:03Z'
message: ReplicaSet "mcm-apiserver-564dfd455" has timed out progressing.
reason: ProgressDeadlineExceeded
status: 'False'
type: Progressing
observedGeneration: 1
replicas: 1
unavailableReplicas: 1
updatedReplicas: 1
- name: configmap-watcher-708df
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:50:24Z'
lastUpdateTime: '2020-06-26T09:50:24Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T09:49:33Z'
lastUpdateTime: '2020-06-26T09:50:24Z'
message: >-
ReplicaSet "configmap-watcher-708df-766b45dc94" has successfully
progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
- name: multicluster-operators-application
status:
availableReplicas: 1
conditions:
- lastTransitionTime: '2020-06-26T09:22:51Z'
lastUpdateTime: '2020-06-26T09:22:51Z'
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: 'True'
type: Available
- lastTransitionTime: '2020-06-26T09:21:12Z'
lastUpdateTime: '2020-06-26T09:22:51Z'
message: >-
ReplicaSet "multicluster-operators-application-7c8dbb89c5" has
successfully progressed.
reason: NewReplicaSetAvailable
status: 'True'
type: Progressing
observedGeneration: 13
readyReplicas: 1
replicas: 1
updatedReplicas: 1
desiredVersion: 1.0.1
phase: Pending
Describe the bug A clear and concise description of what the bug is.
To Reproduce
Follow the readme at https://github.com/open-cluster-management/deploy#prepare-to-deploy-open-cluster-management-instance-only-do-once
I think the major reason is mongodb failed to start caused the mcm apiserver failed to start.
I was using snapshot 2.0.0-SNAPSHOT-2020-06-23-14-20-27
Expected behavior A clear and concise description of what you expected to happen.
Screenshots
Desktop (please complete the following information):
Additional context Add any other context about the problem here.