Closed dszortyka closed 5 years ago
vrutkovs
commented
5 years ago What's the output of oc get csr and docker logs k8s_api_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_1?
Also usermod -aG docker origin is a bad idea but not related to this failure
dszortyka
commented
5 years ago Thanks Vadim, appreciate your input. As I'm learning, trying to make it work first while reading the docs. This is my 4 week trying to deploy with so different options, finally decided to keep it clean and ask for assistance.
[root@okd-cluster cni]# oc get csr
NAME AGE REQUESTOR CONDITION
csr-4db4z 16h system:node:okd-cluster.home.net Pending
csr-4js9k 17h system:node:okd-cluster.home.net Pending
csr-5jlgn 51m system:node:okd-cluster.home.net Pending
csr-6qs9h 1h system:node:okd-cluster.home.net Pending
csr-6wwmj 15h system:node:okd-cluster.home.net Pending
csr-74fx4 18h system:node:okd-cluster.home.net Pending
csr-7qctz 18h system:node:okd-cluster.home.net Pending
csr-8lhhc 17h system:node:okd-cluster.home.net Pending
csr-9mn4j 20h system:node:okd-cluster.home.net Pending
csr-9rl42 1h system:node:okd-cluster.home.net Pending
csr-b9j6g 20h system:node:okd-cluster.home.net Pending
csr-chr76 21h system:node:okd-cluster.home.net Pending
csr-csrpv 25m system:node:okd-cluster.home.net Pending
csr-cxvcn 15h system:node:okd-cluster.home.net Pending
csr-d2j9h 14h system:node:okd-cluster.home.net Pending
csr-dv6ck 21h system:node:okd-cluster.home.net Pending
csr-f4qlb 21h system:admin Pending
csr-fv9rz 15h system:node:okd-cluster.home.net Pending
csr-gb6gp 13h system:node:okd-cluster.home.net Pending
csr-h924j 21h system:node:okd-cluster.home.net Pending
csr-hdv9p 19h system:node:okd-cluster.home.net Pending
csr-hqg8t 14h system:node:okd-cluster.home.net Pending
csr-j9p9t 12m system:node:okd-cluster.home.net Pending
csr-jbcw2 19h system:node:okd-cluster.home.net Pending
csr-jcbsd 17h system:node:okd-cluster.home.net Pending
csr-jr8r5 14h system:node:okd-cluster.home.net Pending
csr-jtxn4 13h system:node:okd-cluster.home.net Pending
csr-jwc6t 16h system:node:okd-cluster.home.net Pending
csr-k4w5r 19h system:node:okd-cluster.home.net Pending
csr-kcg9x 38m system:node:okd-cluster.home.net Pending
csr-n5g45 15h system:node:okd-cluster.home.net Pending
csr-pcrc7 19h system:node:okd-cluster.home.net Pending
csr-pqrkj 16h system:node:okd-cluster.home.net Pending
csr-pwbzq 20h system:node:okd-cluster.home.net Pending
csr-q7qmm 16h system:node:okd-cluster.home.net Pending
csr-qv4bh 17h system:node:okd-cluster.home.net Pending
csr-sfrjg 19h system:node:okd-cluster.home.net Pending
csr-tdktk 14h system:node:okd-cluster.home.net Pending
csr-ths9r 20h system:node:okd-cluster.home.net Pending
csr-vjrnn 21h system:node:okd-cluster.home.net Pending
csr-vxjxg 20h system:node:okd-cluster.home.net Pending
csr-vxskj 16h system:node:okd-cluster.home.net Pending
csr-wc79n 18h system:node:okd-cluster.home.net Pending
csr-wj5qb 1h system:node:okd-cluster.home.net Pending
csr-wmvbx 1h system:node:okd-cluster.home.net Pending
csr-xjdcb 14h system:node:okd-cluster.home.net Pending
csr-xsnzr 14h system:node:okd-cluster.home.net Pending
csr-z46xb 17h system:node:okd-cluster.home.net Pending
csr-z5b57 18h system:node:okd-cluster.home.net Pending
csr-znqpk 6m system:node:okd-cluster.home.net Pending
[root@okd-cluster net.d]# docker logs k8s_api_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_1
E0916 12:25:06.322464 1 helpers.go:134] Encountered config error json: unknown field "masterCount" in object *config.MasterConfig, raw JSON:
{"admissionConfig":{"pluginConfig":{"BuildDefaults":{"configuration":{"apiVersion":"v1","env":[],"kind":"BuildDefaultsConfig","resources":{"limits":{},"requests":{}}}},"BuildOverrides":{"configuration":{"apiVersion":"v1","kind":"BuildOverridesConfig"}},"openshift.io/ImagePolicy":{"configuration":{"apiVersion":"v1","executionRules":[{"matchImageAnnotations":[{"key":"images.openshift.io/deny-execution","value":"true"}],"name":"execution-denied","onResources":[{"resource":"pods"},{"resource":"builds"}],"reject":true,"skipOnResolutionFailure":true}],"kind":"ImagePolicyConfig"}}}},"aggregatorConfig":{"proxyClientInfo":{"certFile":"aggregator-front-proxy.crt","keyFile":"aggregator-front-proxy.key"}},"apiLevels":["v1"],"apiVersion":"v1","authConfig":{"requestHeader":{"clientCA":"front-proxy-ca.crt","clientCommonNames":["aggregator-front-proxy"],"extraHeaderPrefixes":["X-Remote-Extra-"],"groupHeaders":["X-Remote-Group"],"usernameHeaders":["X-Remote-User"]}},"controllerConfig":{"election":{"lockName":"openshift-master-controllers"},"serviceServingCert":{"signer":{"certFile":"service-signer.crt","keyFile":"service-signer.key"}}},"controllers":"*","corsAllowedOrigins":["(?i)//127\\.0\\.0\\.1(:|\\z)","(?i)//localhost(:|\\z)","(?i)//10\\.0\\.0\\.90(:|\\z)","(?i)//kubernetes\\.default(:|\\z)","(?i)//kubernetes\\.default\\.svc\\.cluster\\.local(:|\\z)","(?i)//kubernetes(:|\\z)","(?i)//openshift\\.default(:|\\z)","(?i)//openshift\\.default\\.svc(:|\\z)","(?i)//okd\\-cluster\\.home\\.net(:|\\z)","(?i)//172\\.30\\.0\\.1(:|\\z)","(?i)//openshift\\.default\\.svc\\.cluster\\.local(:|\\z)","(?i)//kubernetes\\.default\\.svc(:|\\z)","(?i)//openshift(:|\\z)"],"dnsConfig":{"bindAddress":"0.0.0.0:8053","bindNetwork":"tcp4"},"etcdClientInfo":{"ca":"master.etcd-ca.crt","certFile":"master.etcd-client.crt","keyFile":"master.etcd-client.key","urls":["https://okd-cluster.home.net:2379"]},"etcdStorageConfig":{"kubernetesStoragePrefix":"kubernetes.io","kubernetesStorageVersion":"v1","openShiftStoragePrefix":"openshift.io","openShiftStorageVersion":"v1"},"imageConfig":{"format":"docker.io/openshift/origin-${component}:${version}","latest":false},"imagePolicyConfig":{"internalRegistryHostname":"docker-registry.default.svc:5000"},"kind":"MasterConfig","kubeletClientInfo":{"ca":"ca-bundle.crt","certFile":"master.kubelet-client.crt","keyFile":"master.kubelet-client.key","port":10250},"kubernetesMasterConfig":{"apiServerArguments":{"storage-backend":["etcd3"],"storage-media-type":["application/vnd.kubernetes.protobuf"]},"controllerArguments":{"cluster-signing-cert-file":["/etc/origin/master/ca.crt"],"cluster-signing-key-file":["/etc/origin/master/ca.key"],"pv-recycler-pod-template-filepath-hostpath":["/etc/origin/master/recycler_pod.yaml"],"pv-recycler-pod-template-filepath-nfs":["/etc/origin/master/recycler_pod.yaml"]},"masterCount":1,"masterIP":"10.0.0.90","podEvictionTimeout":null,"proxyClientInfo":{"certFile":"master.proxy-client.crt","keyFile":"master.proxy-client.key"},"schedulerArguments":null,"schedulerConfigFile":"/etc/origin/master/scheduler.json","servicesNodePortRange":"","servicesSubnet":"172.30.0.0/16","staticNodeNames":[]},"masterClients":{"externalKubernetesClientConnectionOverrides":{"acceptContentTypes":"application/vnd.kubernetes.protobuf,application/json","burst":400,"contentType":"application/vnd.kubernetes.protobuf","qps":200},"externalKubernetesKubeConfig":"","openshiftLoopbackClientConnectionOverrides":{"acceptContentTypes":"application/vnd.kubernetes.protobuf,application/json","burst":600,"contentType":"application/vnd.kubernetes.protobuf","qps":300},"openshiftLoopbackKubeConfig":"openshift-master.kubeconfig"},"masterPublicURL":"https://okd-cluster.home.net:8443","networkConfig":{"clusterNetworks":[{"cidr":"10.128.0.0/14","hostSubnetLength":9}],"externalIPNetworkCIDRs":["0.0.0.0/0"],"networkPluginName":"redhat/openshift-ovs-subnet","serviceNetworkCIDR":"172.30.0.0/16"},"oauthConfig":{"assetPublicURL":"https://okd-cluster.home.net:8443/console/","grantConfig":{"method":"auto"},"identityProviders":[{"challenge":true,"login":true,"mappingMethod":"claim","name":"htpasswd_auth","provider":{"apiVersion":"v1","file":"/etc/origin/master/htpasswd","kind":"HTPasswdPasswordIdentityProvider"}}],"masterCA":"ca-bundle.crt","masterPublicURL":"https://okd-cluster.home.net:8443","masterURL":"https://okd-cluster.home.net:8443","sessionConfig":{"sessionMaxAgeSeconds":3600,"sessionName":"ssn","sessionSecretsFile":"/etc/origin/master/session-secrets.yaml"},"tokenConfig":{"accessTokenMaxAgeSeconds":86400,"authorizeTokenMaxAgeSeconds":500}},"pauseControllers":false,"policyConfig":{"bootstrapPolicyFile":"/etc/origin/master/policy.json","openshiftInfrastructureNamespace":"openshift-infra","openshiftSharedResourcesNamespace":"openshift"},"projectConfig":{"defaultNodeSelector":"node-role.kubernetes.io/compute=true","projectRequestMessage":"","projectRequestTemplate":"","securityAllocator":{"mcsAllocatorRange":"s0:/2","mcsLabelsPerProject":5,"uidAllocatorRange":"1000000000-1999999999/10000"}},"routingConfig":{"subdomain":"apps.home.net"},"serviceAccountConfig":{"limitSecretReferences":false,"managedNames":["default","builder","deployer"],"masterCA":"ca-bundle.crt","privateKeyFile":"serviceaccounts.private.key","publicKeyFiles":["serviceaccounts.public.key"]},"servingInfo":{"bindAddress":"0.0.0.0:8443","bindNetwork":"tcp4","certFile":"master.server.crt","clientCA":"ca.crt","keyFile":"master.server.key","maxRequestsInFlight":500,"requestTimeoutSeconds":3600},"volumeConfig":{"dynamicProvisioningEnabled":true}}
I0916 12:25:06.332212 1 plugins.go:84] Registered admission plugin "NamespaceLifecycle"
I0916 12:25:06.332225 1 plugins.go:84] Registered admission plugin "Initializers"
I0916 12:25:06.332230 1 plugins.go:84] Registered admission plugin "ValidatingAdmissionWebhook"
I0916 12:25:06.332235 1 plugins.go:84] Registered admission plugin "MutatingAdmissionWebhook"
I0916 12:25:06.332239 1 plugins.go:84] Registered admission plugin "AlwaysAdmit"
I0916 12:25:06.332242 1 plugins.go:84] Registered admission plugin "AlwaysPullImages"
I0916 12:25:06.332427 1 plugins.go:84] Registered admission plugin "LimitPodHardAntiAffinityTopology"
I0916 12:25:06.332436 1 plugins.go:84] Registered admission plugin "DefaultTolerationSeconds"
I0916 12:25:06.332440 1 plugins.go:84] Registered admission plugin "AlwaysDeny"
I0916 12:25:06.332446 1 plugins.go:84] Registered admission plugin "EventRateLimit"
I0916 12:25:06.332451 1 plugins.go:84] Registered admission plugin "DenyEscalatingExec"
I0916 12:25:06.332455 1 plugins.go:84] Registered admission plugin "DenyExecOnPrivileged"
I0916 12:25:06.332459 1 plugins.go:84] Registered admission plugin "ExtendedResourceToleration"
I0916 12:25:06.332480 1 plugins.go:84] Registered admission plugin "OwnerReferencesPermissionEnforcement"
I0916 12:25:06.332486 1 plugins.go:84] Registered admission plugin "ImagePolicyWebhook"
I0916 12:25:06.332675 1 plugins.go:84] Registered admission plugin "LimitRanger"
I0916 12:25:06.332684 1 plugins.go:84] Registered admission plugin "NamespaceAutoProvision"
I0916 12:25:06.332689 1 plugins.go:84] Registered admission plugin "NamespaceExists"
I0916 12:25:06.332693 1 plugins.go:84] Registered admission plugin "NodeRestriction"
I0916 12:25:06.332697 1 plugins.go:84] Registered admission plugin "PersistentVolumeLabel"
I0916 12:25:06.332702 1 plugins.go:84] Registered admission plugin "PodNodeSelector"
I0916 12:25:06.332707 1 plugins.go:84] Registered admission plugin "PodPreset"
I0916 12:25:06.332711 1 plugins.go:84] Registered admission plugin "PodTolerationRestriction"
I0916 12:25:06.332732 1 plugins.go:84] Registered admission plugin "ResourceQuota"
I0916 12:25:06.332924 1 plugins.go:84] Registered admission plugin "PodSecurityPolicy"
I0916 12:25:06.332934 1 plugins.go:84] Registered admission plugin "Priority"
I0916 12:25:06.332940 1 plugins.go:84] Registered admission plugin "SecurityContextDeny"
I0916 12:25:06.332953 1 plugins.go:84] Registered admission plugin "ServiceAccount"
I0916 12:25:06.332961 1 plugins.go:84] Registered admission plugin "DefaultStorageClass"
I0916 12:25:06.332969 1 plugins.go:84] Registered admission plugin "PersistentVolumeClaimResize"
I0916 12:25:06.332974 1 plugins.go:84] Registered admission plugin "StorageObjectInUseProtection"
I0916 12:25:10.891793 1 plugins.go:84] Registered admission plugin "NamespaceLifecycle"
I0916 12:25:10.891817 1 plugins.go:84] Registered admission plugin "Initializers"
I0916 12:25:10.891822 1 plugins.go:84] Registered admission plugin "ValidatingAdmissionWebhook"
I0916 12:25:10.891826 1 plugins.go:84] Registered admission plugin "MutatingAdmissionWebhook"
I0916 12:25:10.891832 1 plugins.go:84] Registered admission plugin "AlwaysAdmit"
I0916 12:25:10.891836 1 plugins.go:84] Registered admission plugin "AlwaysPullImages"
I0916 12:25:10.891840 1 plugins.go:84] Registered admission plugin "LimitPodHardAntiAffinityTopology"
I0916 12:25:10.891845 1 plugins.go:84] Registered admission plugin "DefaultTolerationSeconds"
I0916 12:25:10.891849 1 plugins.go:84] Registered admission plugin "AlwaysDeny"
I0916 12:25:10.891855 1 plugins.go:84] Registered admission plugin "EventRateLimit"
I0916 12:25:10.891859 1 plugins.go:84] Registered admission plugin "DenyEscalatingExec"
I0916 12:25:10.891863 1 plugins.go:84] Registered admission plugin "DenyExecOnPrivileged"
I0916 12:25:10.891867 1 plugins.go:84] Registered admission plugin "ExtendedResourceToleration"
I0916 12:25:10.891871 1 plugins.go:84] Registered admission plugin "OwnerReferencesPermissionEnforcement"
I0916 12:25:10.891877 1 plugins.go:84] Registered admission plugin "ImagePolicyWebhook"
I0916 12:25:10.891881 1 plugins.go:84] Registered admission plugin "LimitRanger"
I0916 12:25:10.891885 1 plugins.go:84] Registered admission plugin "NamespaceAutoProvision"
I0916 12:25:10.891889 1 plugins.go:84] Registered admission plugin "NamespaceExists"
I0916 12:25:10.891893 1 plugins.go:84] Registered admission plugin "NodeRestriction"
I0916 12:25:10.891898 1 plugins.go:84] Registered admission plugin "PersistentVolumeLabel"
I0916 12:25:10.891903 1 plugins.go:84] Registered admission plugin "PodNodeSelector"
I0916 12:25:10.891907 1 plugins.go:84] Registered admission plugin "PodPreset"
I0916 12:25:10.891911 1 plugins.go:84] Registered admission plugin "PodTolerationRestriction"
I0916 12:25:10.891915 1 plugins.go:84] Registered admission plugin "ResourceQuota"
I0916 12:25:10.891919 1 plugins.go:84] Registered admission plugin "PodSecurityPolicy"
I0916 12:25:10.892234 1 plugins.go:84] Registered admission plugin "Priority"
I0916 12:25:10.892248 1 plugins.go:84] Registered admission plugin "SecurityContextDeny"
I0916 12:25:10.892258 1 plugins.go:84] Registered admission plugin "ServiceAccount"
I0916 12:25:10.892263 1 plugins.go:84] Registered admission plugin "DefaultStorageClass"
I0916 12:25:10.892268 1 plugins.go:84] Registered admission plugin "PersistentVolumeClaimResize"
I0916 12:25:10.892272 1 plugins.go:84] Registered admission plugin "StorageObjectInUseProtection"
I0916 12:25:11.387479 1 register.go:151] Admission plugin AlwaysAdmit is not enabled. It will not be started.
I0916 12:25:11.387882 1 register.go:151] Admission plugin NamespaceAutoProvision is not enabled. It will not be started.
I0916 12:25:11.388169 1 register.go:151] Admission plugin NamespaceExists is not enabled. It will not be started.
I0916 12:25:11.388480 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: NamespaceLifecycle.
I0916 12:25:11.388750 1 register.go:151] Admission plugin EventRateLimit is not enabled. It will not be started.
I0916 12:25:11.389017 1 register.go:151] Admission plugin ProjectRequestLimit is not enabled. It will not be started.
I0916 12:25:11.389327 1 register.go:151] Admission plugin openshift.io/RestrictSubjectBindings is not enabled. It will not be started.
I0916 12:25:11.389662 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: openshift.io/JenkinsBootstrapper.
I0916 12:25:11.389965 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: openshift.io/BuildConfigSecretInjector.
I0916 12:25:11.390291 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: BuildByStrategy.
I0916 12:25:11.390588 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: openshift.io/ImageLimitRange.
I0916 12:25:11.390621 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: openshift.io/ImageLimitRange.
I0916 12:25:11.390868 1 register.go:151] Admission plugin RunOnceDuration is not enabled. It will not be started.
I0916 12:25:11.391151 1 register.go:151] Admission plugin PodNodeConstraints is not enabled. It will not be started.
I0916 12:25:11.391428 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: OriginPodNodeEnvironment.
I0916 12:25:11.391884 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: PodNodeSelector.
I0916 12:25:11.391930 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: PodNodeSelector.
I0916 12:25:11.392218 1 register.go:151] Admission plugin ClusterResourceOverride is not enabled. It will not be started.
I0916 12:25:11.392720 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: ExternalIPRanger.
I0916 12:25:11.395647 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: openshift.io/RestrictedEndpointsAdmission.
I0916 12:25:11.406483 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: openshift.io/ImagePolicy.
I0916 12:25:11.406506 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: openshift.io/ImagePolicy.
I0916 12:25:11.407417 1 register.go:151] Admission plugin ImagePolicyWebhook is not enabled. It will not be started.
I0916 12:25:11.407697 1 register.go:151] Admission plugin PodPreset is not enabled. It will not be started.
I0916 12:25:11.407978 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: LimitRanger.
I0916 12:25:11.407990 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: LimitRanger.
I0916 12:25:11.408339 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: ServiceAccount.
I0916 12:25:11.408372 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: ServiceAccount.
I0916 12:25:11.408646 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: NodeRestriction.
I0916 12:25:11.408889 1 register.go:151] Admission plugin SecurityContextDeny is not enabled. It will not be started.
I0916 12:25:11.409221 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: SecurityContextConstraint.
I0916 12:25:11.409254 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: SecurityContextConstraint.
I0916 12:25:11.409494 1 register.go:151] Admission plugin PodSecurityPolicy is not enabled. It will not be started.
I0916 12:25:11.409768 1 register.go:151] Admission plugin DenyEscalatingExec is not enabled. It will not be started.
I0916 12:25:11.410088 1 register.go:151] Admission plugin DenyExecOnPrivileged is not enabled. It will not be started.
I0916 12:25:11.410365 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: DefaultStorageClass.
I0916 12:25:11.410640 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: PersistentVolumeClaimResize.
I0916 12:25:11.410897 1 register.go:151] Admission plugin AlwaysPullImages is not enabled. It will not be started.
I0916 12:25:11.411164 1 register.go:151] Admission plugin LimitPodHardAntiAffinityTopology is not enabled. It will not be started.
I0916 12:25:11.411431 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: SCCExecRestrictions.
W0916 12:25:11.411693 1 admission.go:71] PersistentVolumeLabel admission controller is deprecated. Please remove this controller from your configuration files and scripts.
I0916 12:25:11.411742 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: PersistentVolumeLabel.
I0916 12:25:11.412023 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: OwnerReferencesPermissionEnforcement.
I0916 12:25:11.412282 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: openshift.io/IngressAdmission.
I0916 12:25:11.412563 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: Priority.
I0916 12:25:11.412576 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: Priority.
I0916 12:25:11.412803 1 register.go:151] Admission plugin ExtendedResourceToleration is not enabled. It will not be started.
I0916 12:25:11.413122 1 register.go:151] Admission plugin DefaultTolerationSeconds is not enabled. It will not be started.
I0916 12:25:11.413386 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: StorageObjectInUseProtection.
I0916 12:25:11.413618 1 register.go:151] Admission plugin Initializers is not enabled. It will not be started.
I0916 12:25:11.413899 1 register.go:151] Admission plugin MutatingAdmissionWebhook is not enabled. It will not be started.
I0916 12:25:11.414628 1 register.go:151] Admission plugin ValidatingAdmissionWebhook is not enabled. It will not be started.
I0916 12:25:11.414930 1 plugins.go:158] Loaded 1 mutating admission controller(s) successfully in the following order: PodTolerationRestriction.
I0916 12:25:11.414941 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: PodTolerationRestriction.
I0916 12:25:11.415172 1 register.go:151] Admission plugin AlwaysDeny is not enabled. It will not be started.
I0916 12:25:11.415668 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: ResourceQuota.
I0916 12:25:11.416004 1 plugins.go:161] Loaded 1 validating admission controller(s) successfully in the following order: openshift.io/ClusterResourceQuota.
I0916 12:25:11.419253 1 master_config.go:414] Initializing cache sizes based on 0MB limit
I0916 12:25:11.419314 1 master_config.go:476] Using the lease endpoint reconciler with TTL=15s and interval=10s
I0916 12:25:11.438195 1 start_master.go:484] Starting master on 0.0.0.0:8443 (v3.11.0+739ab57-285)
I0916 12:25:11.438213 1 start_master.go:485] Public master address is https://okd-cluster.home.net:8443
I0916 12:25:11.438222 1 start_master.go:489] Using images from "docker.io/openshift/origin-<component>:v3.11.0"
W0916 12:25:11.438261 1 dns_server.go:37] Binding DNS on port 8053 instead of 53, which may not be resolvable from all clients
W0916 12:25:11.438719 1 server.go:79] Unable to keep dnsmasq up to date, 0.0.0.0:8053 must point to port 53
I0916 12:25:11.438794 1 logs.go:49] skydns: ready for queries on cluster.local. for tcp4://0.0.0.0:8053 [rcache 0]
I0916 12:25:11.438801 1 logs.go:49] skydns: ready for queries on cluster.local. for udp4://0.0.0.0:8053 [rcache 0]
I0916 12:25:11.542491 1 dns_server.go:63] DNS listening at 0.0.0.0:8053
I0916 12:25:11.552788 1 plugins.go:84] Registered admission plugin "NamespaceLifecycle"
I0916 12:25:11.552806 1 plugins.go:84] Registered admission plugin "Initializers"
I0916 12:25:11.552811 1 plugins.go:84] Registered admission plugin "ValidatingAdmissionWebhook"
I0916 12:25:11.552816 1 plugins.go:84] Registered admission plugin "MutatingAdmissionWebhook"
I0916 12:25:11.552822 1 plugins.go:84] Registered admission plugin "AlwaysAdmit"
I0916 12:25:11.552827 1 plugins.go:84] Registered admission plugin "AlwaysPullImages"
I0916 12:25:11.552831 1 plugins.go:84] Registered admission plugin "LimitPodHardAntiAffinityTopology"
I0916 12:25:11.552836 1 plugins.go:84] Registered admission plugin "DefaultTolerationSeconds"
I0916 12:25:11.552840 1 plugins.go:84] Registered admission plugin "AlwaysDeny"
I0916 12:25:11.552846 1 plugins.go:84] Registered admission plugin "EventRateLimit"
I0916 12:25:11.552854 1 plugins.go:84] Registered admission plugin "DenyEscalatingExec"
I0916 12:25:11.552858 1 plugins.go:84] Registered admission plugin "DenyExecOnPrivileged"
I0916 12:25:11.552863 1 plugins.go:84] Registered admission plugin "ExtendedResourceToleration"
I0916 12:25:11.552867 1 plugins.go:84] Registered admission plugin "OwnerReferencesPermissionEnforcement"
I0916 12:25:11.553073 1 plugins.go:84] Registered admission plugin "ImagePolicyWebhook"
I0916 12:25:11.553087 1 plugins.go:84] Registered admission plugin "LimitRanger"
I0916 12:25:11.553092 1 plugins.go:84] Registered admission plugin "NamespaceAutoProvision"
I0916 12:25:11.553097 1 plugins.go:84] Registered admission plugin "NamespaceExists"
I0916 12:25:11.553101 1 plugins.go:84] Registered admission plugin "NodeRestriction"
I0916 12:25:11.553106 1 plugins.go:84] Registered admission plugin "PersistentVolumeLabel"
I0916 12:25:11.553110 1 plugins.go:84] Registered admission plugin "PodNodeSelector"
I0916 12:25:11.553115 1 plugins.go:84] Registered admission plugin "PodPreset"
I0916 12:25:11.553120 1 plugins.go:84] Registered admission plugin "PodTolerationRestriction"
I0916 12:25:11.553125 1 plugins.go:84] Registered admission plugin "ResourceQuota"
I0916 12:25:11.553129 1 plugins.go:84] Registered admission plugin "PodSecurityPolicy"
I0916 12:25:11.553134 1 plugins.go:84] Registered admission plugin "Priority"
I0916 12:25:11.553138 1 plugins.go:84] Registered admission plugin "SecurityContextDeny"
I0916 12:25:11.553145 1 plugins.go:84] Registered admission plugin "ServiceAccount"
I0916 12:25:11.553150 1 plugins.go:84] Registered admission plugin "DefaultStorageClass"
I0916 12:25:11.553155 1 plugins.go:84] Registered admission plugin "PersistentVolumeClaimResize"
I0916 12:25:11.553159 1 plugins.go:84] Registered admission plugin "StorageObjectInUseProtection"
I0916 12:25:11.553679 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:11.566718 1 store.go:1397] Monitoring customresourcedefinitions.apiextensions.k8s.io count at <storage-prefix>//apiextensions.k8s.io/customresourcedefinitions
I0916 12:25:11.605461 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:11.617517 1 store.go:1397] Monitoring securitycontextconstraints.security.openshift.io count at <storage-prefix>//securitycontextconstraints
I0916 12:25:11.617733 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:11.629283 1 store.go:1397] Monitoring deploymentconfigs.apps.openshift.io count at <storage-prefix>//deploymentconfigs
W0916 12:25:11.760599 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:11.760661 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:11 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:11 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:11.845841 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:11.856284 1 store.go:1397] Monitoring rolebindingrestrictions.authorization.openshift.io count at <storage-prefix>//rolebindingrestrictions
W0916 12:25:11.995046 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:11.995119 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:11 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:11 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:12.080715 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:12.091991 1 store.go:1397] Monitoring builds.build.openshift.io count at <storage-prefix>//builds
I0916 12:25:12.101038 1 store.go:1397] Monitoring buildconfigs.build.openshift.io count at <storage-prefix>//buildconfigs
W0916 12:25:12.200488 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:12.200580 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:12.296484 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:12.334077 1 store.go:1397] Monitoring images.image.openshift.io count at <storage-prefix>//images
I0916 12:25:12.342976 1 store.go:1397] Monitoring imagestreams.image.openshift.io count at <storage-prefix>//imagestreams
W0916 12:25:12.475772 1 genericapiserver.go:346] Skipping API image.openshift.io/1.0 because it has no resources.
W0916 12:25:12.475834 1 genericapiserver.go:346] Skipping API image.openshift.io/pre012 because it has no resources.
W0916 12:25:12.481690 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:12.481750 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:12.573355 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:12.586102 1 store.go:1397] Monitoring hostsubnets.network.openshift.io count at <storage-prefix>//registry/sdnsubnets
I0916 12:25:12.598785 1 store.go:1397] Monitoring netnamespaces.network.openshift.io count at <storage-prefix>//registry/sdnnetnamespaces
I0916 12:25:12.619091 1 store.go:1397] Monitoring clusternetworks.network.openshift.io count at <storage-prefix>//registry/sdnnetworks
I0916 12:25:12.628506 1 store.go:1397] Monitoring egressnetworkpolicies.network.openshift.io count at <storage-prefix>//registry/egressnetworkpolicy
W0916 12:25:12.668431 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:12.668592 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:12.753613 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:12.766851 1 store.go:1397] Monitoring oauthclients.oauth.openshift.io count at <storage-prefix>//oauth/clients
I0916 12:25:12.776359 1 store.go:1397] Monitoring oauthauthorizetokens.oauth.openshift.io count at <storage-prefix>//oauth/authorizetokens
I0916 12:25:12.784974 1 store.go:1397] Monitoring oauthaccesstokens.oauth.openshift.io count at <storage-prefix>//oauth/accesstokens
I0916 12:25:12.793213 1 store.go:1397] Monitoring oauthclientauthorizations.oauth.openshift.io count at <storage-prefix>//oauth/clientauthorizations
W0916 12:25:12.831658 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:12.831730 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:12.930412 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
W0916 12:25:12.978678 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
W0916 12:25:12.979011 1 swagger.go:37] No API exists for predefined swagger description /api/v1
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:12 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:13.015438 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:13.026152 1 store.go:1397] Monitoring clusterresourcequotas.quota.openshift.io count at <storage-prefix>//clusterresourcequotas
W0916 12:25:13.081795 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:13.081837 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:13.147586 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:13.160937 1 store.go:1397] Monitoring routes.route.openshift.io count at <storage-prefix>//routes
W0916 12:25:13.186472 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:13.186783 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:13.246684 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:13.257294 1 store.go:1397] Monitoring rangeallocations.security.openshift.io count at <storage-prefix>//rangeallocations
W0916 12:25:13.364404 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:13.364463 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:13.460280 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:13.472014 1 store.go:1397] Monitoring templates.template.openshift.io count at <storage-prefix>//templates
I0916 12:25:13.482524 1 store.go:1397] Monitoring templateinstances.template.openshift.io count at <storage-prefix>//templateinstances
I0916 12:25:13.490758 1 store.go:1397] Monitoring brokertemplateinstances.template.openshift.io count at <storage-prefix>//brokertemplateinstances
W0916 12:25:13.546391 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:13.546988 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:13.655995 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:13.666526 1 store.go:1397] Monitoring users.user.openshift.io count at <storage-prefix>//users
I0916 12:25:13.674696 1 store.go:1397] Monitoring identities.user.openshift.io count at <storage-prefix>//useridentities
I0916 12:25:13.683058 1 store.go:1397] Monitoring groups.user.openshift.io count at <storage-prefix>//groups
W0916 12:25:13.768901 1 swagger.go:37] No API exists for predefined swagger description /api/v1
W0916 12:25:13.769069 1 swagger.go:37] No API exists for predefined swagger description /oapi/v1
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:13 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:13.827926 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:14.528319 1 openshift_apiserver.go:566] Started Origin API at /oapi/v1
W0916 12:25:14.550515 1 swagger.go:37] No API exists for predefined swagger description /api/v1
[restful] 2019/09/16 12:25:14 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:14 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:14.993174 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:15.003629 1 store.go:1397] Monitoring podtemplates count at <storage-prefix>//podtemplates
I0916 12:25:15.012327 1 store.go:1397] Monitoring events count at <storage-prefix>//events
I0916 12:25:15.020203 1 store.go:1397] Monitoring limitranges count at <storage-prefix>//limitranges
I0916 12:25:15.028402 1 store.go:1397] Monitoring resourcequotas count at <storage-prefix>//resourcequotas
I0916 12:25:15.037007 1 store.go:1397] Monitoring secrets count at <storage-prefix>//secrets
I0916 12:25:15.045576 1 store.go:1397] Monitoring persistentvolumes count at <storage-prefix>//persistentvolumes
I0916 12:25:15.054207 1 store.go:1397] Monitoring persistentvolumeclaims count at <storage-prefix>//persistentvolumeclaims
I0916 12:25:15.063859 1 store.go:1397] Monitoring configmaps count at <storage-prefix>//configmaps
I0916 12:25:15.072399 1 store.go:1397] Monitoring namespaces count at <storage-prefix>//namespaces
I0916 12:25:15.081172 1 store.go:1397] Monitoring endpoints count at <storage-prefix>//services/endpoints
I0916 12:25:15.090534 1 store.go:1397] Monitoring nodes count at <storage-prefix>//minions
I0916 12:25:15.102569 1 store.go:1397] Monitoring pods count at <storage-prefix>//pods
I0916 12:25:15.111764 1 store.go:1397] Monitoring serviceaccounts count at <storage-prefix>//serviceaccounts
I0916 12:25:15.119938 1 store.go:1397] Monitoring services count at <storage-prefix>//services/specs
I0916 12:25:15.150264 1 store.go:1397] Monitoring replicationcontrollers count at <storage-prefix>//controllers
W0916 12:25:15.710236 1 genericapiserver.go:346] Skipping API 1.0 because it has no resources.
W0916 12:25:15.710571 1 genericapiserver.go:346] Skipping API pre012 because it has no resources.
I0916 12:25:15.710723 1 master.go:418] Enabling API group "authentication.k8s.io".
I0916 12:25:15.710867 1 master.go:418] Enabling API group "authorization.k8s.io".
I0916 12:25:15.719853 1 store.go:1397] Monitoring horizontalpodautoscalers.autoscaling count at <storage-prefix>//horizontalpodautoscalers
I0916 12:25:15.728382 1 store.go:1397] Monitoring horizontalpodautoscalers.autoscaling count at <storage-prefix>//horizontalpodautoscalers
I0916 12:25:15.728403 1 master.go:418] Enabling API group "autoscaling".
I0916 12:25:15.737983 1 store.go:1397] Monitoring jobs.batch count at <storage-prefix>//jobs
I0916 12:25:15.746558 1 store.go:1397] Monitoring cronjobs.batch count at <storage-prefix>//cronjobs
I0916 12:25:15.746576 1 master.go:418] Enabling API group "batch".
I0916 12:25:15.754975 1 store.go:1397] Monitoring certificatesigningrequests.certificates.k8s.io count at <storage-prefix>//certificatesigningrequests
I0916 12:25:15.754993 1 master.go:418] Enabling API group "certificates.k8s.io".
I0916 12:25:15.764107 1 store.go:1397] Monitoring replicationcontrollers count at <storage-prefix>//controllers
I0916 12:25:15.776164 1 store.go:1397] Monitoring daemonsets.extensions count at <storage-prefix>//daemonsets
I0916 12:25:15.784916 1 store.go:1397] Monitoring deployments.extensions count at <storage-prefix>//deployments
I0916 12:25:15.794301 1 store.go:1397] Monitoring ingresses.extensions count at <storage-prefix>//ingress
I0916 12:25:15.803252 1 store.go:1397] Monitoring podsecuritypolicies.policy count at <storage-prefix>//podsecuritypolicy
I0916 12:25:15.811521 1 store.go:1397] Monitoring replicasets.extensions count at <storage-prefix>//replicasets
I0916 12:25:15.827062 1 store.go:1397] Monitoring networkpolicies.networking.k8s.io count at <storage-prefix>//networkpolicies
I0916 12:25:15.827365 1 master.go:418] Enabling API group "extensions".
I0916 12:25:15.837237 1 store.go:1397] Monitoring networkpolicies.networking.k8s.io count at <storage-prefix>//networkpolicies
I0916 12:25:15.837255 1 master.go:418] Enabling API group "networking.k8s.io".
I0916 12:25:15.851430 1 store.go:1397] Monitoring poddisruptionbudgets.policy count at <storage-prefix>//poddisruptionbudgets
I0916 12:25:15.861107 1 store.go:1397] Monitoring podsecuritypolicies.policy count at <storage-prefix>//podsecuritypolicy
I0916 12:25:15.861377 1 master.go:418] Enabling API group "policy".
I0916 12:25:15.869416 1 store.go:1397] Monitoring roles.rbac.authorization.k8s.io count at <storage-prefix>//roles
I0916 12:25:15.877637 1 store.go:1397] Monitoring rolebindings.rbac.authorization.k8s.io count at <storage-prefix>//rolebindings
I0916 12:25:15.886693 1 store.go:1397] Monitoring clusterroles.rbac.authorization.k8s.io count at <storage-prefix>//clusterroles
I0916 12:25:15.895450 1 store.go:1397] Monitoring clusterrolebindings.rbac.authorization.k8s.io count at <storage-prefix>//clusterrolebindings
I0916 12:25:15.904004 1 store.go:1397] Monitoring roles.rbac.authorization.k8s.io count at <storage-prefix>//roles
I0916 12:25:15.912386 1 store.go:1397] Monitoring rolebindings.rbac.authorization.k8s.io count at <storage-prefix>//rolebindings
I0916 12:25:15.921164 1 store.go:1397] Monitoring clusterroles.rbac.authorization.k8s.io count at <storage-prefix>//clusterroles
I0916 12:25:15.930164 1 store.go:1397] Monitoring clusterrolebindings.rbac.authorization.k8s.io count at <storage-prefix>//clusterrolebindings
I0916 12:25:15.930298 1 master.go:418] Enabling API group "rbac.authorization.k8s.io".
I0916 12:25:15.939252 1 store.go:1397] Monitoring priorityclasses.scheduling.k8s.io count at <storage-prefix>//priorityclasses
I0916 12:25:15.939270 1 master.go:418] Enabling API group "scheduling.k8s.io".
I0916 12:25:15.939280 1 master.go:410] Skipping disabled API group "settings.k8s.io".
I0916 12:25:15.948220 1 store.go:1397] Monitoring storageclasses.storage.k8s.io count at <storage-prefix>//storageclasses
I0916 12:25:15.957262 1 store.go:1397] Monitoring volumeattachments.storage.k8s.io count at <storage-prefix>//volumeattachments
I0916 12:25:15.966146 1 store.go:1397] Monitoring storageclasses.storage.k8s.io count at <storage-prefix>//storageclasses
I0916 12:25:15.966443 1 master.go:418] Enabling API group "storage.k8s.io".
I0916 12:25:15.975394 1 store.go:1397] Monitoring deployments.extensions count at <storage-prefix>//deployments
I0916 12:25:15.984441 1 store.go:1397] Monitoring statefulsets.apps count at <storage-prefix>//statefulsets
I0916 12:25:15.993399 1 store.go:1397] Monitoring controllerrevisions.apps count at <storage-prefix>//controllerrevisions
I0916 12:25:16.002318 1 store.go:1397] Monitoring deployments.extensions count at <storage-prefix>//deployments
I0916 12:25:16.010783 1 store.go:1397] Monitoring statefulsets.apps count at <storage-prefix>//statefulsets
I0916 12:25:16.019298 1 store.go:1397] Monitoring daemonsets.extensions count at <storage-prefix>//daemonsets
I0916 12:25:16.027701 1 store.go:1397] Monitoring replicasets.extensions count at <storage-prefix>//replicasets
I0916 12:25:16.035777 1 store.go:1397] Monitoring controllerrevisions.apps count at <storage-prefix>//controllerrevisions
I0916 12:25:16.044085 1 store.go:1397] Monitoring deployments.extensions count at <storage-prefix>//deployments
I0916 12:25:16.052661 1 store.go:1397] Monitoring statefulsets.apps count at <storage-prefix>//statefulsets
I0916 12:25:16.061881 1 store.go:1397] Monitoring daemonsets.extensions count at <storage-prefix>//daemonsets
I0916 12:25:16.072353 1 store.go:1397] Monitoring replicasets.extensions count at <storage-prefix>//replicasets
I0916 12:25:16.080820 1 store.go:1397] Monitoring controllerrevisions.apps count at <storage-prefix>//controllerrevisions
I0916 12:25:16.080837 1 master.go:418] Enabling API group "apps".
I0916 12:25:16.089432 1 store.go:1397] Monitoring validatingwebhookconfigurations.admissionregistration.k8s.io count at <storage-prefix>//validatingwebhookconfigurations
I0916 12:25:16.098151 1 store.go:1397] Monitoring mutatingwebhookconfigurations.admissionregistration.k8s.io count at <storage-prefix>//mutatingwebhookconfigurations
I0916 12:25:16.098167 1 master.go:418] Enabling API group "admissionregistration.k8s.io".
I0916 12:25:16.106510 1 store.go:1397] Monitoring events count at <storage-prefix>//events
I0916 12:25:16.106528 1 master.go:418] Enabling API group "events.k8s.io".
W0916 12:25:16.397803 1 genericapiserver.go:346] Skipping API batch/v2alpha1 because it has no resources.
W0916 12:25:16.770260 1 genericapiserver.go:346] Skipping API rbac.authorization.k8s.io/v1alpha1 because it has no resources.
W0916 12:25:16.777834 1 genericapiserver.go:346] Skipping API scheduling.k8s.io/v1alpha1 because it has no resources.
W0916 12:25:16.799724 1 genericapiserver.go:346] Skipping API storage.k8s.io/v1alpha1 because it has no resources.
W0916 12:25:17.397155 1 genericapiserver.go:346] Skipping API admissionregistration.k8s.io/v1alpha1 because it has no resources.
[restful] 2019/09/16 12:25:17 log.go:33: [restful/swagger] listing is available at https://okd-cluster.home.net:8443/swaggerapi
[restful] 2019/09/16 12:25:17 log.go:33: [restful/swagger] https://okd-cluster.home.net:8443/swaggerui/ is mapped to folder /swagger-ui/
I0916 12:25:18.588385 1 patch_handlerchain.go:91] Starting OAuth2 API at /oauth
I0916 12:25:18.601538 1 store.go:1397] Monitoring apiservices.apiregistration.k8s.io count at <storage-prefix>//apiservices
I0916 12:25:18.611358 1 store.go:1397] Monitoring apiservices.apiregistration.k8s.io count at <storage-prefix>//apiservices
I0916 12:25:22.316210 1 serve.go:116] Serving securely on 0.0.0.0:8443
I0916 12:25:22.317151 1 available_controller.go:283] Starting AvailableConditionController
I0916 12:25:22.317160 1 cache.go:32] Waiting for caches to sync for AvailableConditionController controller
I0916 12:25:22.317614 1 crd_finalizer.go:242] Starting CRDFinalizer
I0916 12:25:22.317637 1 openshift_apiserver.go:681] Using default project node label selector: node-role.kubernetes.io/compute=true
I0916 12:25:22.317942 1 apiservice_controller.go:90] Starting APIServiceRegistrationController
I0916 12:25:22.317948 1 cache.go:32] Waiting for caches to sync for APIServiceRegistrationController controller
I0916 12:25:22.317958 1 controller.go:84] Starting OpenAPI AggregationController
I0916 12:25:22.320065 1 clusterquotamapping.go:160] Starting ClusterQuotaMappingController controller
I0916 12:25:22.325739 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43062: EOF
I0916 12:25:22.328357 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43066: EOF
I0916 12:25:22.335182 1 customresource_discovery_controller.go:199] Starting DiscoveryController
I0916 12:25:22.335205 1 naming_controller.go:284] Starting NamingConditionController
I0916 12:25:22.335215 1 establishing_controller.go:73] Starting EstablishingController
I0916 12:25:22.336456 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43068: EOF
I0916 12:25:22.339187 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43070: EOF
I0916 12:25:22.341823 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43072: EOF
I0916 12:25:22.344704 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43076: EOF
I0916 12:25:22.349955 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43078: EOF
I0916 12:25:22.352573 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43080: EOF
I0916 12:25:22.355778 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43082: EOF
I0916 12:25:22.358322 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43084: EOF
I0916 12:25:22.360813 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43086: EOF
I0916 12:25:22.363672 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43092: EOF
I0916 12:25:22.366160 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43100: EOF
I0916 12:25:22.368949 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43102: EOF
I0916 12:25:22.371732 1 logs.go:49] http: TLS handshake error from 10.0.0.90:43104: EOF
I0916 12:25:22.382925 1 crdregistration_controller.go:112] Starting crd-autoregister controller
I0916 12:25:22.383007 1 controller_utils.go:1025] Waiting for caches to sync for crd-autoregister controller
I0916 12:25:22.504909 1 controller_utils.go:1032] Caches are synced for crd-autoregister controller
I0916 12:25:22.505020 1 autoregister_controller.go:136] Starting autoregister controller
I0916 12:25:22.505027 1 cache.go:32] Waiting for caches to sync for autoregister controller
I0916 12:25:22.523712 1 cache.go:39] Caches are synced for APIServiceRegistrationController controller
I0916 12:25:22.534738 1 cache.go:39] Caches are synced for AvailableConditionController controller
W0916 12:25:22.551020 1 lease_endpoint_reconciler.go:176] Resetting endpoints for master service "kubernetes" to [10.0.0.90]
I0916 12:25:22.580214 1 controller.go:597] quota admission added evaluator for: { events}
I0916 12:25:22.605689 1 cache.go:39] Caches are synced for autoregister controller
I0916 12:25:23.345518 1 storage_scheduling.go:100] all system priority classes are created successfully or already exist.
I0916 12:25:23.348890 1 controller.go:597] quota admission added evaluator for: { serviceaccounts}
I0916 12:30:15.098554 1 controller.go:597] quota admission added evaluator for: {rbac.authorization.k8s.io rolebindings}
I0916 12:30:15.101829 1 controller.go:597] quota admission added evaluator for: { serviceaccounts}
I0916 12:30:15.102086 1 controller.go:597] quota admission added evaluator for: {rbac.authorization.k8s.io rolebindings}
I0916 12:30:15.175156 1 controller.go:597] quota admission added evaluator for: { namespaces}
I0916 12:30:15.178354 1 controller.go:597] quota admission added evaluator for: { namespaces}
I0916 12:30:15.203693 1 controller.go:597] quota admission added evaluator for: {apps.openshift.io deploymentconfigs}
I0916 12:30:15.204266 1 controller.go:597] quota admission added evaluator for: {apps.openshift.io deploymentconfigs}
I0916 12:31:29.858851 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55486: EOF
I0916 12:31:30.080787 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55488: EOF
I0916 12:31:30.271259 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55490: EOF
I0916 12:31:30.496494 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55491: EOF
I0916 12:31:30.679460 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55492: EOF
I0916 12:31:30.899335 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55493: EOF
I0916 12:31:31.129958 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55494: EOF
I0916 12:31:34.647650 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55498: EOF
I0916 12:31:35.957946 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55505: EOF
I0916 12:31:36.193957 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55506: EOF
I0916 12:31:36.423963 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55507: EOF
I0916 12:31:36.686425 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55508: EOF
I0916 12:31:36.885152 1 logs.go:49] http: TLS handshake error from 10.0.0.104:55509: EOF
I0916 12:32:20.279111 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.282368 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.287416 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.292539 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.297055 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.301348 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.306034 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.310273 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.367511 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
I0916 12:32:20.369926 1 httplog.go:64] &{system:openshift-master [system:masters system:authenticated] map[]} is acting as &{system:serviceaccount:kube-system:namespace-controller [system:serviceaccounts system:serviceaccounts:kube-system system:authenticated] map[]}
E0916 12:35:11.967959 1 watcher.go:208] watch chan error: etcdserver: mvcc: required revision has been compacted
...
E0916 14:08:13.528384 1 watcher.go:208] watch chan error: etcdserver: mvcc: required revision has been compactedUpd: Attaching files to make it easier. docker_logs k8s_api_master-api-okd-cluster.home.net_kube-system.txt ocr_logs.txt
vrutkovs
commented
5 years ago Right, so nodes are sending csr's but these don't get approved.
Could you include journalctl -b -l -u origin-node from any of the nodes?
dszortyka
commented
5 years ago ETCD doesn't seem to have errors..
Also, from : docker logs k8s_api_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_1
E0916 12:25:06.322464 1 helpers.go:134] Encountered config error json: unknown field "masterCount" in object *config.MasterConfig, raw JSON:
{"admissionConfig":{"pluginConfig":{"BuildDefaults":{"configuration":{"apiVersion":"v1","env":[],"kind":"BuildDefaultsConfig","resources":{"limits":{},"requests":{}}}},"BuildOverrides":{"configuration":{"apiVersion":"v1","kind":"BuildOverridesConfig"}},"openshift.io/ImagePolicy":{"configuration":{"apiVersion":"v1","executionRules":[{"matchImageAnnotations":[{"key":"images.openshift.io/deny-execution","value":"true"}],"name":"execution-denied","onResources":[{"resource":"pods"},{"resource":"builds"}],"reject":true,"skipOnResolutionFailure":true}],"kind":"ImagePolicyConfig"}}}},"aggregatorConfig":{"proxyClientInfo":{"certFile":"aggregator-front-proxy.crt","keyFile":"aggregator-front-proxy.key"}},"apiLevels":["v1"],"apiVersion":"v1","authConfig":{"requestHeader":{"clientCA":"front-proxy-ca.crt","clientCommonNames":["aggregator-front-proxy"],"extraHeaderPrefixes":["X-Remote-Extra-"],"groupHeaders":["X-Remote-Group"],"usernameHeaders":["X-Remote-User"]}},"controllerConfig":{"election":{"lockName":"openshift-master-controllers"},"serviceServingCert":{"signer":{"certFile":"service-signer.crt","keyFile":"service-signer.key"}}},"controllers":"*","corsAllowedOrigins":["(?i)//127\\.0\\.0\\.1(:|\\z)","(?i)//localhost(:|\\z)","(?i)//10\\.0\\.0\\.90(:|\\z)","(?i)//kubernetes\\.default(:|\\z)","(?i)//kubernetes\\.default\\.svc\\.cluster\\.local(:|\\z)","(?i)//kubernetes(:|\\z)","(?i)//openshift\\.default(:|\\z)","(?i)//openshift\\.default\\.svc(:|\\z)","(?i)//okd\\-cluster\\.home\\.net(:|\\z)","(?i)//172\\.30\\.0\\.1(:|\\z)","(?i)//openshift\\.default\\.svc\\.cluster\\.local(:|\\z)","(?i)//kubernetes\\.default\\.svc(:|\\z)","(?i)//openshift(:|\\z)"],"dnsConfig":{"bindAddress":"0.0.0.0:8053","bindNetwork":"tcp4"},"etcdClientInfo":{"ca":"master.etcd-ca.crt","certFile":"master.etcd-client.crt","keyFile":"master.etcd-client.key","urls":["https://okd-cluster.home.net:2379"]},"etcdStorageConfig":{"kubernetesStoragePrefix":"kubernetes.io","kubernetesStorageVersion":"v1","openShiftStoragePrefix":"openshift.io","openShiftStorageVersion":"v1"},"imageConfig":{"format":"docker.io/openshift/origin-${component}:${version}","latest":false},"imagePolicyConfig":{"internalRegistryHostname":"docker-registry.default.svc:5000"},"kind":"MasterConfig","kubeletClientInfo":{"ca":"ca-bundle.crt","certFile":"master.kubelet-client.crt","keyFile":"master.kubelet-client.key","port":10250},"kubernetesMasterConfig":{"apiServerArguments":{"storage-backend":["etcd3"],"storage-media-type":["application/vnd.kubernetes.protobuf"]},"controllerArguments":{"cluster-signing-cert-file":["/etc/origin/master/ca.crt"],"cluster-signing-key-file":["/etc/origin/master/ca.key"],"pv-recycler-pod-template-filepath-hostpath":["/etc/origin/master/recycler_pod.yaml"],"pv-recycler-pod-template-filepath-nfs":["/etc/origin/master/recycler_pod.yaml"]},"masterCount":1,"masterIP":"10.0.0.90","podEvictionTimeout":null,"proxyClientInfo":{"certFile":"master.proxy-client.crt","keyFile":"master.proxy-client.key"},"schedulerArguments":null,"schedulerConfigFile":"/etc/origin/master/scheduler.json","servicesNodePortRange":"","servicesSubnet":"172.30.0.0/16","staticNodeNames":[]},"masterClients":{"externalKubernetesClientConnectionOverrides":{"acceptContentTypes":"application/vnd.kubernetes.protobuf,application/json","burst":400,"contentType":"application/vnd.kubernetes.protobuf","qps":200},"externalKubernetesKubeConfig":"","openshiftLoopbackClientConnectionOverrides":{"acceptContentTypes":"application/vnd.kubernetes.protobuf,application/json","burst":600,"contentType":"application/vnd.kubernetes.protobuf","qps":300},"openshiftLoopbackKubeConfig":"openshift-master.kubeconfig"},"masterPublicURL":"https://okd-cluster.home.net:8443","networkConfig":{"clusterNetworks":[{"cidr":"10.128.0.0/14","hostSubnetLength":9}],"externalIPNetworkCIDRs":["0.0.0.0/0"],"networkPluginName":"redhat/openshift-ovs-subnet","serviceNetworkCIDR":"172.30.0.0/16"},"oauthConfig":{"assetPublicURL":"https://okd-cluster.home.net:8443/console/","grantConfig":{"method":"auto"},"identityProviders":[{"challenge":true,"login":true,"mappingMethod":"claim","name":"htpasswd_auth","provider":{"apiVersion":"v1","file":"/etc/origin/master/htpasswd","kind":"HTPasswdPasswordIdentityProvider"}}],"masterCA":"ca-bundle.crt","masterPublicURL":"https://okd-cluster.home.net:8443","masterURL":"https://okd-cluster.home.net:8443","sessionConfig":{"sessionMaxAgeSeconds":3600,"sessionName":"ssn","sessionSecretsFile":"/etc/origin/master/session-secrets.yaml"},"tokenConfig":{"accessTokenMaxAgeSeconds":86400,"authorizeTokenMaxAgeSeconds":500}},"pauseControllers":false,"policyConfig":{"bootstrapPolicyFile":"/etc/origin/master/policy.json","openshiftInfrastructureNamespace":"openshift-infra","openshiftSharedResourcesNamespace":"openshift"},"projectConfig":{"defaultNodeSelector":"node-role.kubernetes.io/compute=true","projectRequestMessage":"","projectRequestTemplate":"","securityAllocator":{"mcsAllocatorRange":"s0:/2","mcsLabelsPerProject":5,"uidAllocatorRange":"1000000000-1999999999/10000"}},"routingConfig":{"subdomain":"apps.home.net"},"serviceAccountConfig":{"limitSecretReferences":false,"managedNames":["default","builder","deployer"],"masterCA":"ca-bundle.crt","privateKeyFile":"serviceaccounts.private.key","publicKeyFiles":["serviceaccounts.public.key"]},"servingInfo":{"bindAddress":"0.0.0.0:8443","bindNetwork":"tcp4","certFile":"master.server.crt","clientCA":"ca.crt","keyFile":"master.server.key","maxRequestsInFlight":500,"requestTimeoutSeconds":3600},"volumeConfig":{"dynamicProvisioningEnabled":true}}See attached. docker logs k8s_etcd_master-etcd-okd-cluster.home.net_kube-system_7219ffc00eb55985caab299da834753a_1.txt
dszortyka
commented
5 years ago Right, so nodes are sending csr's but these don't get approved.
Could you include
journalctl -b -l -u origin-nodefrom any of the nodes?
Got no results filtering, attaching the full logs from one of the nodes. journalctl.txt
dszortyka
commented
5 years ago [root@okd-cluster net.d]# systemctl status origin-node.service ● origin-node.service - OpenShift Node Loaded: loaded (/etc/systemd/system/origin-node.service; enabled; vendor preset: disabled) Active: active (running) since Mon 2019-09-16 11:01:32 -03; 35min ago Docs: https://github.com/openshift/origin Main PID: 18401 (hyperkube) Memory: 88.7M CGroup: /system.slice/origin-node.service └─18401 /usr/bin/hyperkube kubelet --v=2 --address=0.0.0.0 --allow-privileged=true --anonymous-auth=true --authentication-token-webhook=true --authentication-token-webhook-cache-ttl=5m --authorization-mode=Webhook --authorization-webhook-cache-authorized-t...
Sep 16 11:36:49 okd-cluster.home.net origin-node[18401]: W0916 11:36:49.838357 18401 cni.go:172] Unable to update cni config: No networks found in /etc/cni/net.d Sep 16 11:36:49 okd-cluster.home.net origin-node[18401]: E0916 11:36:49.839232 18401 kubelet.go:2101] Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
vrutkovs
commented
5 years ago >Installed: origin-node-3.11.0-1.el7.git.0.62803d0.x86_64
Seems CentOS PaaS repos are broken - these are the packages from last year. @DanyC97 do you know if there has been a change recently?
vrutkovs
commented
5 years ago Meanwhile could you try reinstalling with this ansible var:
openshift_additional_repos: [
{"name": "origin-pr", "baseurl": "https://rpms.svc.ci.openshift.org/openshift-origin-v3.11/", "enabled": 1, "gpgcheck": 0},
{"name": "origin-pr-dependencies", "baseurl": "http://mirror.centos.org/centos/7/paas/x86_64/openshift-origin/", "enabled": 1, "gpgcheck": 0}
]?
dszortyka
commented
5 years ago No go.
However, this time I see different packages installed. Attached the second try with -vvv option.
changed: [okd-cluster.home.net] => {
"attempts": 3,
"changed": true,
"changes": {
"installed": [
"origin-node-3.11*",
"origin-clients-3.11*"
]
},
"invocation": {
"module_args": {
"allow_downgrade": false,
"autoremove": false,
"bugfix": false,
"conf_file": null,
"disable_excludes": null,
"disable_gpg_check": false,
"disable_plugin": [],
"disablerepo": [],
"download_dir": null,
"download_only": false,
"enable_plugin": [],
"enablerepo": [],
"exclude": [],
"install_repoquery": true,
"install_weak_deps": true,
"installroot": "/",
"list": null,
"lock_timeout": 30,
"name": [
"origin-node-3.11*",
"origin-clients-3.11*",
"conntrack-tools"
],
"releasever": null,
"security": false,
"skip_broken": false,
"state": "present",
"update_cache": false,
"update_only": false,
"use_backend": "auto",
"validate_certs": true
}
},
"msg": "",
"rc": 0,
"results": [
"conntrack-tools-1.4.4-4.el7.x86_64 providing conntrack-tools is already installed",
"Loaded plugins: fastestmirror\nLoading mirror speeds from cached hostfile\n * base: centos.brnet.net.br\n * centos-ansible26: centos.xpg.com.br\n * epel: mirror.globo.com\n * extras: centos.brnet.net.br\n * updates: centos.brnet.net.br\nResolving Dependencies\n--> Running transaction check\n---> Package origin-clients.x86_64 0:3.11.0-1.285.739ab57 will be installed\n---> Package origin-node.x86_64 0:3.11.0-1.285.739ab57 will be installed\n--> Processing Dependency: origin-hyperkube = 3.11.0-1.285.739ab57 for package: origin-node-3.11.0-1.285.739ab57.x86_64\n--> Running transaction check\n---> Package origin-hyperkube.x86_64 0:3.11.0-1.285.739ab57 will be installed\n--> Finished Dependency Resolution\n\nDependencies Resolved\n\n================================================================================\n Package Arch Version Repository Size\n================================================================================\nInstalling:\n origin-clients x86_64 3.11.0-1.285.739ab57 origin-pr 22 M\n origin-node x86_64 3.11.0-1.285.739ab57 origin-pr 6.1 M\nInstalling for dependencies:\n origin-hyperkube x86_64 3.11.0-1.285.739ab57 origin-pr 34 M\n\nTransaction Summary\n================================================================================\nInstall 2 Packages (+1 Dependent package)\n\nTotal size: 63 M\nTotal download size: 57 M\nInstalled size: 386 M\nDownloading packages:\nDelta RPMs disabled because /usr/bin/applydeltarpm not installed.\n--------------------------------------------------------------------------------\nTotal 149 kB/s | 57 MB 06:29 \nRunning transaction check\nRunning transaction test\nTransaction test succeeded\nRunning transaction\n Installing : origin-hyperkube-3.11.0-1.285.739ab57.x86_64 1/3 \n Installing : origin-node-3.11.0-1.285.739ab57.x86_64 2/3 \n Installing : origin-clients-3.11.0-1.285.739ab57.x86_64 3/3 \n Verifying : origin-node-3.11.0-1.285.739ab57.x86_64 1/3 \n Verifying : origin-clients-3.11.0-1.285.739ab57.x86_64 2/3 \n Verifying : origin-hyperkube-3.11.0-1.285.739ab57.x86_64 3/3 \n\nInstalled:\n origin-clients.x86_64 0:3.11.0-1.285.739ab57 \n origin-node.x86_64 0:3.11.0-1.285.739ab57 \n\nDependency Installed:\n origin-hyperkube.x86_64 0:3.11.0-1.285.739ab57 \n\nComplete!\n"
]
}It progresses though:
Attempting to register node okd-cluster.home.net", is a good sign. Seems etcd still has a lot of unsigneds csrs
Could you try uninstalling it and deploying again?
dszortyka
commented
5 years ago Sorry for my delay, it didn't work, with the same results..
docker ps:
[root@okd-cluster master]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
7be9bbc1917f 7ef3a92939de "/bin/bash -c '#!/..." About an hour ago Up About an hour k8s_controllers_master-controllers-okd-cluster.home.net_kube-system_e4b71f87f964c37acaebc9ddbcb9b08d_0
fe06a963c51a docker.io/openshift/origin-pod:v3.11.0 "/usr/bin/pod" About an hour ago Up About an hour k8s_POD_master-controllers-okd-cluster.home.net_kube-system_e4b71f87f964c37acaebc9ddbcb9b08d_0
ce72a6fb30f1 7ef3a92939de "/bin/bash -c '#!/..." About an hour ago Up About an hour k8s_api_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_0
3c81c17146ff docker.io/openshift/origin-pod:v3.11.0 "/usr/bin/pod" About an hour ago Up About an hour k8s_POD_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_0
acde4ccf0326 ff5dd2137a4f "/bin/sh -c '#!/bi..." About an hour ago Up About an hour k8s_etcd_master-etcd-okd-cluster.home.net_kube-system_7219ffc00eb55985caab299da834753a_0
2c66fd23b7f5 docker.io/openshift/origin-pod:v3.11.0 "/usr/bin/pod" About an hour ago Up About an hour k8s_POD_master-etcd-okd-cluster.home.net_kube-system_7219ffc00eb55985caab299da834753a_0Pending CSRs:
[root@okd-cluster master]# oc get csr
NAME AGE REQUESTOR CONDITION
csr-7dw2l 1h system:node:okd-cluster.home.net Pending
csr-8gfrm 1h system:node:okd-cluster.home.net Pending
csr-9pqtr 1h system:node:okd-cluster.home.net Pending
csr-hrbqw 13m system:node:okd-cluster.home.net Pending
csr-j4jbt 1h system:node:okd-cluster.home.net Pending
csr-j8dq2 2h system:admin Pending
csr-k4x6d 1h system:node:okd-cluster.home.net Pending
csr-l98rs 52m system:node:okd-cluster.home.net Pending
csr-mvjvb 26m system:node:okd-cluster.home.net Pending
csr-pkgr2 1h system:node:okd-cluster.home.net Pending
csr-s2mq4 56s system:node:okd-cluster.home.net Pending
csr-tf68r 39m system:node:okd-cluster.home.net Pending
From k8s_controllers_master-controllers-okd-cluster.home.net_kube-system_e4b71f87f964c37acaebc9ddbcb9b08d_0:
I0917 21:13:56.096128 1 controllermanager.go:128] Version: v1.11.0+d4cacc0
I0917 21:13:56.096169 1 leaderelection.go:185] attempting to acquire leader lease kube-system/kube-controller-manager...
E0917 21:14:06.087264 1 leaderelection.go:234] error retrieving resource lock kube-system/openshift-master-controllers: Get https://okd-cluster.home.net:8443/api/v1/namespaces/kube-system/configmaps/openshift-master-controllers: net/http: TLS handshake timeout
E0917 21:14:06.087341 1 reflector.go:136] k8s.io/kubernetes/cmd/kube-scheduler/app/server.go:176: Failed to list *v1.Pod: Get https://okd-cluster.home.net:8443/api/v1/pods?fieldSelector=status.phase%21%3DFailed%2Cstatus.phase%21%3DSucceeded&limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097281 1 leaderelection.go:234] error retrieving resource lock kube-system/kube-controller-manager: Get https://okd-cluster.home.net:8443/api/v1/namespaces/kube-system/configmaps/kube-controller-manager: net/http: TLS handshake timeout
E0917 21:14:06.097374 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1beta1.PodDisruptionBudget: Get https://okd-cluster.home.net:8443/apis/policy/v1beta1/poddisruptionbudgets?limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097417 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1beta1.StatefulSet: Get https://okd-cluster.home.net:8443/apis/apps/v1beta1/statefulsets?limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097452 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1.Node: Get https://okd-cluster.home.net:8443/api/v1/nodes?limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097495 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1.ReplicationController: Get https://okd-cluster.home.net:8443/api/v1/replicationcontrollers?limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097527 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1.Service: Get https://okd-cluster.home.net:8443/api/v1/services?limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097565 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1.PersistentVolume: Get https://okd-cluster.home.net:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097600 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1.StorageClass: Get https://okd-cluster.home.net:8443/apis/storage.k8s.io/v1/storageclasses?limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097630 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1.PersistentVolumeClaim: Get https://okd-cluster.home.net:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0: net/http: TLS handshake timeout
E0917 21:14:06.097661 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1beta1.ReplicaSet: Get https://okd-cluster.home.net:8443/apis/extensions/v1beta1/replicasets?limit=500&resourceVersion=0: net/http: TLS handshake timeout
I0917 21:14:09.996659 1 controller_utils.go:1025] Waiting for caches to sync for scheduler controller
I0917 21:14:10.096736 1 controller_utils.go:1032] Caches are synced for scheduler controller
I0917 21:14:10.096757 1 leaderelection.go:185] attempting to acquire leader lease kube-system/kube-scheduler...
I0917 21:14:10.167509 1 leaderelection.go:194] successfully acquired lease kube-system/openshift-master-controllers
[root@okd-node1 ~]# rpm -aq|grep origin
origin-hyperkube-3.11.0-1.285.739ab57.x86_64
origin-excluder-3.11.0-1.285.739ab57.noarch
origin-docker-excluder-3.11.0-1.285.739ab57.noarch
origin-clients-3.11.0-1.285.739ab57.x86_64
centos-release-openshift-origin311-1-2.el7.centos.noarch
origin-node-3.11.0-1.285.739ab57.x86_64
[root@okd-cluster ~]# rpm -aq|grep origin
origin-hyperkube-3.11.0-1.285.739ab57.x86_64
origin-3.11.0-1.285.739ab57.x86_64
origin-excluder-3.11.0-1.285.739ab57.noarch
origin-docker-excluder-3.11.0-1.285.739ab57.noarch
origin-clients-3.11.0-1.285.739ab57.x86_64
centos-release-openshift-origin311-1-2.el7.centos.noarch
origin-node-3.11.0-1.285.739ab57.x86_64
Attached full log files from nodes.
[deploy_3.txt.zip] (https://github.com/openshift/openshift-ansible/files/3623740/deploy_3.txt.zip)
[k8s_etcd_master-etcd-okd-cluster.home.net_kube-system_7219ffc00eb55985caab299da834753a_0.txt] (https://github.com/openshift/openshift-ansible/files/3623741/k8s_etcd_master-etcd-okd-cluster.home.net_kube-system_7219ffc00eb55985caab299da834753a_0.txt)
[k8s_api_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_0.txt] (https://github.com/openshift/openshift-ansible/files/3623742/k8s_api_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_0.txt)
[k8s_controllers_master-controllers-okd-cluster.home.net_kube-system_e4b71f87f964c37acaebc9ddbcb9b08d_0.txt] (https://github.com/openshift/openshift-ansible/files/3623743/k8s_controllers_master-controllers-okd-cluster.home.net_kube-system_e4b71f87f964c37acaebc9ddbcb9b08d_0.txt)
[oc adm diagnostics.txt] (https://github.com/openshift/openshift-ansible/files/3623755/oc.adm.diagnostics.txt)
vudhaya
commented
5 years ago Hi, I'm getting the same errors.
vrutkovs
commented
5 years ago It seems some firewall or network setup is breaking connections:
I0917 21:14:25.238485 1 node_lifecycle_controller.go:558] Controller observed a new Node: "okd-cluster.home.net"
I0917 21:14:25.238515 1 controller_utils.go:212] Recording Registered Node okd-cluster.home.net in Controller event message for node okd-cluster.home.net
I0917 21:14:25.238547 1 node_lifecycle_controller.go:1095] Initializing eviction metric for zone:
W0917 21:14:25.238597 1 node_lifecycle_controller.go:782] Missing timestamp for Node okd-cluster.home.net. Assuming now as a timestamp.
I0917 21:14:25.238624 1 node_lifecycle_controller.go:945] Controller detected that all Nodes are not-Ready. Entering master disruption mode.in controllers and
I0917 21:14:08.847404 1 logs.go:49] http: TLS handshake error from 10.0.0.90:47652: EOFDid you run prerequisites playbook? Could you try running it again (keep the output) and run deploy playbook again?
dszortyka
commented
5 years ago Thanks for your help!!
While I try to reinstall it, here's the uninstall and pre-reqs logs.
Also, FYI, here's a bit more of my landscape.
I suspect on this topics, but I'm a beginner on this:
hostname
[root@okd-cluster ~]# hostname
okd-cluster.home.net
[root@okd-cluster ~]# hostname -f
okd-cluster.home.net
[root@okd-cluster ~]# hostname -A
okd-cluster.home.net okd-cluster.home.net
/etc/hosts file from okd-cluster host (Cluster)
[root@okd-cluster ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
#10.0.0.90 okd-cluster.home.net okd-cluster
#192.168.10.90 okd-cluster.int.home.net
#10.0.0.91 okd-node1.home.net okd-node1
#192.168.10.91 okd-node1.int.home.net
#10.0.0.92 okd-node2.home.net okd-node2
#10.0.0.93 okd-infranode1.home.net okd-infranode1
#192.168.10.93 okd-node1.int.home.net
#10.0.0.94 okd-infranode2.home.net okd-infranode2Resolv.conf (before OC installation)
[root@okd-cluster ~]# cat /etc/resolv.conf
# Generated by NetworkManager
search home.net
nameserver 10.0.0.99/etc/named.conf (from DNS server - 10.0.0.99)
options {
listen-on port 53 { 127.0.0.1; 10.0.0.99; };
forwarders {
8.8.8.8;
8.8.4.4;
};
// allow-query { localhost; 192.168.10.1/24; 172.17.0.0/24; 172.30.0.0/16; 10.0.0.1/24; 10.0.0.90; 10.0.0.91; 10.0.0.93; 10.0.0.0/24; };
allow-query { any; };
/* listen-on-v6 port 53 { ::1; }; */
directory "/var/named";
dump-file "/var/named/data/cache_dump.db";
statistics-file "/var/named/data/named_stats.txt";
memstatistics-file "/var/named/data/named_mem_stats.txt";
//recursing-file "/var/named/data/named.recursing";
//secroots-file "/var/named/data/named.secroots";
recursion yes;
forward only;
dnssec-enable yes;
dnssec-validation yes;
/* Path to ISC DLV key */
bindkeys-file "/etc/named.iscdlv.key";
managed-keys-directory "/var/named/dynamic";
pid-file "/run/named/named.pid";
session-keyfile "/run/named/session.key";
};
logging {
channel default_debug {
file "data/named.run";
severity dynamic;
};
};
zone "." IN {
type hint;
file "named.ca";
};
zone "home.net" IN {
type master;
file "home.net.db";
allow-update { none; };
};
zone "0.0.10.in-addr.arpa" IN {
type master;
file "home.net.db.rev";
allow-update { none; };
};Domain (home.net , configured for learning purposes)
[root@dns ~]# cat /var/named/home.net.db
$TTL 86400
@ IN SOA dns.home.net. root.home.net. (
1031
3H ;Refresh
15M ;Retry
1W ;Expire
1D ;Minimum TTL
)
;Name Server Information
@ IN NS dns.home.net.
@ IN A 10.0.0.90
@ IN A 10.0.0.91
@ IN A 10.0.0.93
;IP address of Name Server
dns IN A 10.0.0.99
*.okd-cluster.home.net. 10 IN A 10.0.0.90
*.apps.home.net. 300 IN A 10.0.0.90
okd-cluster IN A 10.0.0.90
okd-node1 IN A 10.0.0.91
okd-infranode1 IN A 10.0.0.93Reverse domain (home.net)
[root@dns ~]# cat /var/named/home.net.db.rev
$TTL 86400
@ IN SOA dns.home.net. root.home.net. (
1031
1H ;Refresh
5M ;Retry
1D ;Expire
1D ;Minimum TTL
)
;Name Server Information
@ IN NS dns.home.net.
@ IN PTR home.net.
dns IN A 10.0.0.99
okd-cluster IN A 10.0.0.90
okd-node1 IN A 10.0.0.91
okd-infranode1 IN A 10.0.0.93
;Reverse lookup for Name Server
99 IN PTR dns.home.net.
;90 IN PTR okd-cluster.home.net.
;PTR Record IP address to HostName
90 IN PTR okd-cluster.home.net.
91 IN PTR okd-node1.home.net.
93 IN PTR okd-infranode1.home.net.[pre_req.txt] (https://github.com/openshift/openshift-ansible/files/3626842/pre_req.txt)
[uninstall.txt] (https://github.com/openshift/openshift-ansible/files/3626843/uninstall.txt)
vrutkovs
commented
5 years ago Did the prerequisites playbook work?
/label needs/info
openshift-ci-robot
commented
5 years ago @vrutkovs: The label(s) /label needs/info cannot be applied. These labels are supported: platform/aws, platform/azure, platform/baremetal, platform/google, platform/libvirt, platform/openstack, ga
dszortyka
commented
5 years ago Hello all, Sorry for my delay. I was finally able to deploy a cluster with master + node + infranode. Sadly, the initial inventory that we have been playing in this topic and all the modifications we attempted did not work. I tried uncountable times.. =( My landscape is supposed to be properly configured, with even a DNS server to avoid issues related to /etc/hosts.
However, I have few considerations and a possible issue.
1- I was able to deploy with the inventory file provided below in this comment. 2- I had to modify a yml file delivered with openshift-ansible to bypass below issue. To be honest, I have no idea what I did, but it worked.
fatal: [okd-clustert.home.net]: FAILED! => {"reason": "'become' is not a valid attribute for a IncludeRole\n\nThe error appears to be in '/root/openshift-ansible-release-3.11/roles/calico/tasks/certs.yml': line 24, column 3, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\n- name: Calico Node | Configure separate Calico etcd and certs\n ^ here\n"}
The parameter "become: yes" was causing this issue, I commented this line and the installation worked properly.
/root/openshift-ansible-release-3.11/roles/calico/tasks/certs.yml
- name: Calico Node | Configure separate Calico etcd and certs
when: use_calico_etcd
#become: yes
include_role:
name: etcd
tasks_from: server_certificates
vars:
etcd_cert_prefix: calico-etcd-
etcd_conf_dir: "{{ calico_etcd_cert_dir }}"
etcd_cert_config_dir: "{{ calico_etcd_cert_dir }}"
etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
etcd_cert_subdir: "calico-etcd-{{ openshift.common.hostname }}"Here's the inventory file that worked for me.
[OSEv3:children]
masters
nodes
etcd
[OSEv3:vars]
ansible_ssh_user=root
ansible_become=true
ansible_sudo=true
openshift_deployment_type=origin
openshift_web_console_install=true
openshift_console_install=true
openshift_install_examples=true
sdn_network_plugin_name=cni
#openshift_use_flannel=true
openshift_use_dnsmasq=false
openshift_use_calico=true
openshift_use_openshift_sdn=false
osm_cluster_network_cidr=192.168.0.0/16
#openshift_additional_repos=[{'name': 'origin-pr', 'baseurl': 'https://rpms.svc.ci.openshift.org/openshift-origin-v3.11/', 'enabled': '1', 'gpgcheck': '0'},{'name': 'origin-pr-dependencies', 'baseurl': 'http://mirror.centos.org/centos/7/paas/x86_64/openshift-origin/', 'enabled': '1', 'gpgcheck': '0'}]
openshift_disable_check=docker_storage,disk_availability,memory_availability
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]
openshift_docker_insecure_registries=192.168.0.0/16,192.168.10.1/24,172.17.0.0/24,172.30.0.0/16,10.0.0.0/24
[masters]
okd-cluster.home.net openshift_schedulable=true containerized=yes
[etcd]
okd-cluster.home.net
[nodes]
okd-cluster.home.net openshift_node_group_name='node-config-master'
okd-node1.home.net openshift_node_group_name='node-config-compute'
#okd-node2.home.net openshift_node_group_name='node-config-compute'
okd-infranode1.home.net openshift_node_group_name='node-config-infra'
#iodk-infranode2.home.net openshift_node_group_name='node-config-infra'
@vrutkovs thanks a lot for your help!!
I believe this topic can be closed, let me know if you want me to run any other installation attempt or collect any kind of logs.
vrutkovs
commented
5 years ago Excellent!
The parameter "become: yes" was causing this issue, I commented this line and the installation worked properly.
I thought all of these have been eradicated by now. Could you file a separate bug for that.
Closing as fixed. The trick was to run prerequisites
/close
openshift-ci-robot
commented
5 years ago @vrutkovs: Closing this issue.
uselessidbr
commented
4 years ago Not sure if I should reopen this or open a new bug but this exactly error still happening while deploying Openshift Origin v3.11 with Calico.
It has nothing to do with the prerequisites playbook.
The only workaround I found is to put comments on "become: yes" flag.
TASK [calico : include_tasks] ***** Sunday 19 January 2020 19:36:36 -0300 (0:00:00.369) 0:13:41.399 **** fatal: [master1.openshift.local]: FAILED! => {"reason": "'become' is not a valid attribute for a IncludeRole\n\nThe error appears to be in '/home/ansible/openshift-ansible/roles/calico/tasks/certs.yml': line 24, column 3, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\n- name: Calico Node | Configure separate Calico etcd and certs\n ^ here\n"} fatal: [master2.openshift.local]: FAILED! => {"reason": "'become' is not a valid attribute for a IncludeRole\n\nThe error appears to be in '/home/ansible/openshift-ansible/roles/calico/tasks/certs.yml': line 24, column 3, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\n- name: Calico Node | Configure separate Calico etcd and certs\n ^ here\n"} fatal: [master3.openshift.local]: FAILED! => {"reason": "'become' is not a valid attribute for a IncludeRole\n\nThe error appears to be in '/home/ansible/openshift-ansible/roles/calico/tasks/certs.yml': line 24, column 3, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\n- name: Calico Node | Configure separate Calico etcd and certs\n ^ here\n"}
Its happening with this release:
commit 2b4133519598b1e9347c768a51912db4ab5027aa Author: AOS Automation Release Team aos-team-art@redhat.com Date: Thu Jan 16 19:35:05 2020 +0000
Automatic commit of package [openshift-ansible] release [3.11.165-1].
Description
Hello guys, Can you try to me give a help please?
I'm facing basically the issues:
fatal: [okd-cluster.home.net]: FAILED! => {"changed": false, "msg": "Control plane pods didn't come up"}cni.go:172] Unable to update cni config: No networks found in /etc/cni/net.d"My installation consists in the following:
Virtualbox running 3 hosts, each with 2GB and 1vCPU, using bridge network.
10.0.0.90 okd-cluster.home.net okd-cluster 10.0.0.91 okd-node1.home.net okd-node1 10.0.0.93 okd-infranode1.home.net okd-infranode1
Version
Steps To Reproduce
Master
Nodes:
Resumed errors:
I also notice the following when I try to open console: https://okd-cluster.home.net:8443/console
##### Additional Information
I've attached few files which I already saw in previous links I checked: 1- oc adm diagnose 2- deploy file with -vvv 3- master-controllers logs 4- inventory files
$ cat /etc/redhat-release)CENTOS_MANTISBT_PROJECT="CentOS-7" CENTOS_MANTISBT_PROJECT_VERSION="7" REDHAT_SUPPORT_PRODUCT="centos" REDHAT_SUPPORT_PRODUCT_VERSION="7"
[root@okd-cluster ~]# docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES ef8ba3987408 7ef3a92939de "/bin/bash -c '#!/..." 19 minutes ago Up 19 minutes k8s_controllers_master-controllers-okd-cluster.home.net_kube-system_e4b71f87f964c37acaebc9ddbcb9b08d_1 b44d8e692f32 ff5dd2137a4f "/bin/sh -c '#!/bi..." 19 minutes ago Up 19 minutes k8s_etcd_master-etcd-okd-cluster.home.net_kube-system_7219ffc00eb55985caab299da834753a_1 348b47dcfcb4 7ef3a92939de "/bin/bash -c '#!/..." 19 minutes ago Up 19 minutes k8s_api_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_1 b3496ef35bcc docker.io/openshift/origin-pod:v3.11.0 "/usr/bin/pod" 19 minutes ago Up 19 minutes k8s_POD_master-etcd-okd-cluster.home.net_kube-system_7219ffc00eb55985caab299da834753a_1 ffb4b5c2a184 docker.io/openshift/origin-pod:v3.11.0 "/usr/bin/pod" 19 minutes ago Up 19 minutes k8s_POD_master-controllers-okd-cluster.home.net_kube-system_e4b71f87f964c37acaebc9ddbcb9b08d_1 230a3dd196ee docker.io/openshift/origin-pod:v3.11.0 "/usr/bin/pod" 19 minutes ago Up 19 minutes k8s_POD_master-api-okd-cluster.home.net_kube-system_60f548cd1d82d290eb6882da121098d3_1
[root@okd-cluster ~]# docker logs ef8ba3987408 E0916 12:25:06.326781 1 helpers.go:134] Encountered config error json: unknown field "masterCount" in object config.MasterConfig, raw JSON: {"admissionConfig":{"pluginConfig":{"BuildDefaults":{"configuration":{"apiVersion":"v1","env":[],"kind":"BuildDefaultsConfig","resources":{"limits":{},"requests":{}}}},"BuildOverrides":{"configuration":{"apiVersion":"v1","kind":"BuildOverridesConfig"}},"openshift.io/ImagePolicy":{"configuration":{"apiVersion":"v1","executionRules":[{"matchImageAnnotations":[{"key":"images.openshift.io/deny-execution","value":"true"}],"name":"execution-denied","onResources":[{"resource":"pods"},{"resource":"builds"}],"reject":true,"skipOnResolutionFailure":true}],"kind":"ImagePolicyConfig"}}}},"aggregatorConfig":{"proxyClientInfo":{"certFile":"aggregator-front-proxy.crt","keyFile":"aggregator-front-proxy.key"}},"apiLevels":["v1"],"apiVersion":"v1","authConfig":{"requestHeader":{"clientCA":"front-proxy-ca.crt","clientCommonNames":["aggregator-front-proxy"],"extraHeaderPrefixes":["X-Remote-Extra-"],"groupHeaders":["X-Remote-Group"],"usernameHeaders":["X-Remote-User"]}},"controllerConfig":{"election":{"lockName":"openshift-master-controllers"},"serviceServingCert":{"signer":{"certFile":"service-signer.crt","keyFile":"service-signer.key"}}},"controllers":"","corsAllowedOrigins":["(?i)//127\.0\.0\.1(:|\z)","(?i)//localhost(:|\z)","(?i)//10\.0\.0\.90(:|\z)","(?i)//kubernetes\.default(:|\z)","(?i)//kubernetes\.default\.svc\.cluster\.local(:|\z)","(?i)//kubernetes(:|\z)","(?i)//openshift\.default(:|\z)","(?i)//openshift\.default\.svc(:|\z)","(?i)//okd\-cluster\.home\.net(:|\z)","(?i)//172\.30\.0\.1(:|\z)","(?i)//openshift\.default\.svc\.cluster\.local(:|\z)","(?i)//kubernetes\.default\.svc(:|\z)","(?i)//openshift(:|\z)"],"dnsConfig":{"bindAddress":"0.0.0.0:8053","bindNetwork":"tcp4"},"etcdClientInfo":{"ca":"master.etcd-ca.crt","certFile":"master.etcd-client.crt","keyFile":"master.etcd-client.key","urls":["https://okd-cluster.home.net:2379"]},"etcdStorageConfig":{"kubernetesStoragePrefix":"kubernetes.io","kubernetesStorageVersion":"v1","openShiftStoragePrefix":"openshift.io","openShiftStorageVersion":"v1"},"imageConfig":{"format":"docker.io/openshift/origin-${component}:${version}","latest":false},"imagePolicyConfig":{"internalRegistryHostname":"docker-registry.default.svc:5000"},"kind":"MasterConfig","kubeletClientInfo":{"ca":"ca-bundle.crt","certFile":"master.kubelet-client.crt","keyFile":"master.kubelet-client.key","port":10250},"kubernetesMasterConfig":{"apiServerArguments":{"storage-backend":["etcd3"],"storage-media-type":["application/vnd.kubernetes.protobuf"]},"controllerArguments":{"cluster-signing-cert-file":["/etc/origin/master/ca.crt"],"cluster-signing-key-file":["/etc/origin/master/ca.key"],"pv-recycler-pod-template-filepath-hostpath":["/etc/origin/master/recycler_pod.yaml"],"pv-recycler-pod-template-filepath-nfs":["/etc/origin/master/recycler_pod.yaml"]},"masterCount":1,"masterIP":"10.0.0.90","podEvictionTimeout":null,"proxyClientInfo":{"certFile":"master.proxy-client.crt","keyFile":"master.proxy-client.key"},"schedulerArguments":null,"schedulerConfigFile":"/etc/origin/master/scheduler.json","servicesNodePortRange":"","servicesSubnet":"172.30.0.0/16","staticNodeNames":[]},"masterClients":{"externalKubernetesClientConnectionOverrides":{"acceptContentTypes":"application/vnd.kubernetes.protobuf,application/json","burst":400,"contentType":"application/vnd.kubernetes.protobuf","qps":200},"externalKubernetesKubeConfig":"","openshiftLoopbackClientConnectionOverrides":{"acceptContentTypes":"application/vnd.kubernetes.protobuf,application/json","burst":600,"contentType":"application/vnd.kubernetes.protobuf","qps":300},"openshiftLoopbackKubeConfig":"openshift-master.kubeconfig"},"masterPublicURL":"https://okd-cluster.home.net:8443","networkConfig":{"clusterNetworks":[{"cidr":"10.128.0.0/14","hostSubnetLength":9}],"externalIPNetworkCIDRs":["0.0.0.0/0"],"networkPluginName":"redhat/openshift-ovs-subnet","serviceNetworkCIDR":"172.30.0.0/16"},"oauthConfig":{"assetPublicURL":"https://okd-cluster.home.net:8443/console/","grantConfig":{"method":"auto"},"identityProviders":[{"challenge":true,"login":true,"mappingMethod":"claim","name":"htpasswd_auth","provider":{"apiVersion":"v1","file":"/etc/origin/master/htpasswd","kind":"HTPasswdPasswordIdentityProvider"}}],"masterCA":"ca-bundle.crt","masterPublicURL":"https://okd-cluster.home.net:8443","masterURL":"https://okd-cluster.home.net:8443","sessionConfig":{"sessionMaxAgeSeconds":3600,"sessionName":"ssn","sessionSecretsFile":"/etc/origin/master/session-secrets.yaml"},"tokenConfig":{"accessTokenMaxAgeSeconds":86400,"authorizeTokenMaxAgeSeconds":500}},"pauseControllers":false,"policyConfig":{"bootstrapPolicyFile":"/etc/origin/master/policy.json","openshiftInfrastructureNamespace":"openshift-infra","openshiftSharedResourcesNamespace":"openshift"},"projectConfig":{"defaultNodeSelector":"node-role.kubernetes.io/compute=true","projectRequestMessage":"","projectRequestTemplate":"","securityAllocator":{"mcsAllocatorRange":"s0:/2","mcsLabelsPerProject":5,"uidAllocatorRange":"1000000000-1999999999/10000"}},"routingConfig":{"subdomain":"apps.home.net"},"serviceAccountConfig":{"limitSecretReferences":false,"managedNames":["default","builder","deployer"],"masterCA":"ca-bundle.crt","privateKeyFile":"serviceaccounts.private.key","publicKeyFiles":["serviceaccounts.public.key"]},"servingInfo":{"bindAddress":"0.0.0.0:8443","bindNetwork":"tcp4","certFile":"master.server.crt","clientCA":"ca.crt","keyFile":"master.server.key","maxRequestsInFlight":500,"requestTimeoutSeconds":3600},"volumeConfig":{"dynamicProvisioningEnabled":true}} ... E0916 12:25:06.395393 1 reflector.go:136] k8s.io/client-go/informers/factory.go:130: Failed to list *v1beta1.PodDisruptionBudget: Get https://okd-cluster.home.net:8443/apis/policy/v1beta1/poddisruptionbudgets?limit=500&resourceVersion=0: dial tcp 10.0.0.90:8443: connect: connection refused