kubesphere / ks-installer

Install KubeSphere on existing Kubernetes cluster
https://kubesphere.io
Apache License 2.0
527 stars 743 forks source link

node-exporter 这个项目容器无法起来 导致监控不可用 #180

Open zy84338719 opened 4 years ago

zy84338719 commented 4 years ago

系统centos7.7 x3 项目部署日志

TASK [ks-istio : istio | Deploy jaeger-production] *****************************
changed: [localhost]

TASK [ks-istio : istio | Create jaeger operator service] ***********************
changed: [localhost]

PLAY RECAP *********************************************************************
localhost                  : ok=152  changed=130  unreachable=0    failed=0    skipped=33   rescued=0    ignored=1

项目容器报错 kubesphere-monitoring-system 容器组 Pod: node-exporter

0/3 nodes are available: 2 node(s) didn't match node selector, 3 node(s) didn't have free ports for the requested pod ports.

如何修复?

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  annotations:
    deprecated.daemonset.template.generation: '1'
    kubectl.kubernetes.io/last-applied-configuration: >
      {"apiVersion":"apps/v1beta2","kind":"DaemonSet","metadata":{"annotations":{},"labels":{"app":"node-exporter"},"name":"node-exporter","namespace":"kubesphere-monitoring-system"},"spec":{"selector":{"matchLabels":{"app":"node-exporter"}},"template":{"metadata":{"labels":{"app":"node-exporter"}},"spec":{"containers":[{"args":["--web.listen-address=127.0.0.1:9100","--path.procfs=/host/proc","--path.sysfs=/host/sys","--collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)","--collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$"],"image":"dockerhub.qingcloud.com/prometheus/node-exporter:ks-v0.16.0","name":"node-exporter","resources":{"limits":{"cpu":"250m","memory":"180Mi"},"requests":{"cpu":"102m","memory":"180Mi"}},"volumeMounts":[{"mountPath":"/host/proc","name":"proc","readOnly":false},{"mountPath":"/host/sys","name":"sys","readOnly":false},{"mountPath":"/host/root","mountPropagation":"HostToContainer","name":"root","readOnly":true}]},{"args":["--secure-listen-address=$(IP):9100","--upstream=http://127.0.0.1:9100/"],"env":[{"name":"IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}],"image":"dockerhub.qingcloud.com/coreos/kube-rbac-proxy:v0.4.1","name":"kube-rbac-proxy","ports":[{"containerPort":9100,"hostPort":9100,"name":"https"}],"resources":{"limits":{"cpu":"20m","memory":"40Mi"},"requests":{"cpu":"10m","memory":"20Mi"}}}],"hostNetwork":true,"hostPID":true,"imagePullSecrets":[{"name":"qingcloud"}],"nodeSelector":{"beta.kubernetes.io/os":"linux"},"securityContext":{"runAsNonRoot":true,"runAsUser":65534},"serviceAccountName":"node-exporter","tolerations":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"key":"dedicated","operator":"Exists"}],"volumes":[{"hostPath":{"path":"/proc"},"name":"proc"},{"hostPath":{"path":"/sys"},"name":"sys"},{"hostPath":{"path":"/"},"name":"root"}]}}}}
  creationTimestamp: '2019-10-05T06:53:38Z'
  generation: 1
  labels:
    app: node-exporter
  name: node-exporter
  namespace: kubesphere-monitoring-system
  resourceVersion: '108592'
  selfLink: >-
    /apis/apps/v1/namespaces/kubesphere-monitoring-system/daemonsets/node-exporter
  uid: 890f47ac-f1ec-4827-9e75-717774ccfaf1
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: node-exporter
  template:
    metadata:
      labels:
        app: node-exporter
    spec:
      containers:
        - args:
            - '--web.listen-address=127.0.0.1:9100'
            - '--path.procfs=/host/proc'
            - '--path.sysfs=/host/sys'
            - >-
              --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)
            - >-
              --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$
          image: 'dockerhub.qingcloud.com/prometheus/node-exporter:ks-v0.16.0'
          imagePullPolicy: IfNotPresent
          name: node-exporter
          resources:
            limits:
              cpu: 250m
              memory: 180Mi
            requests:
              cpu: 102m
              memory: 180Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /host/proc
              name: proc
            - mountPath: /host/sys
              name: sys
            - mountPath: /host/root
              mountPropagation: HostToContainer
              name: root
              readOnly: true
        - args:
            - '--secure-listen-address=$(IP):9100'
            - '--upstream=http://127.0.0.1:9100/'
          env:
            - name: IP
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP
          image: 'dockerhub.qingcloud.com/coreos/kube-rbac-proxy:v0.4.1'
          imagePullPolicy: IfNotPresent
          name: kube-rbac-proxy
          ports:
            - containerPort: 9100
              hostPort: 9100
              name: https
              protocol: TCP
          resources:
            limits:
              cpu: 20m
              memory: 40Mi
            requests:
              cpu: 10m
              memory: 20Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      hostNetwork: true
      hostPID: true
      imagePullSecrets:
        - name: qingcloud
      nodeSelector:
        beta.kubernetes.io/os: linux
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        runAsNonRoot: true
        runAsUser: 65534
      serviceAccount: node-exporter
      serviceAccountName: node-exporter
      terminationGracePeriodSeconds: 30
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
        - key: dedicated
          operator: Exists
      volumes:
        - hostPath:
            path: /proc
            type: ''
          name: proc
        - hostPath:
            path: /sys
            type: ''
          name: sys
        - hostPath:
            path: /
            type: ''
          name: root
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
status:
  currentNumberScheduled: 3
  desiredNumberScheduled: 3
  numberMisscheduled: 0
  numberReady: 0
  numberUnavailable: 3
  observedGeneration: 1
  updatedNumberScheduled: 3

---
apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: >
      {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"k8s-app":"node-exporter"},"name":"node-exporter","namespace":"kubesphere-monitoring-system"},"spec":{"clusterIP":"None","ports":[{"name":"https","port":9100,"targetPort":"https"}],"selector":{"app":"node-exporter"}}}
  creationTimestamp: '2019-10-05T06:53:38Z'
  labels:
    k8s-app: node-exporter
  name: node-exporter
  namespace: kubesphere-monitoring-system
  resourceVersion: '20957'
  selfLink: /api/v1/namespaces/kubesphere-monitoring-system/services/node-exporter
  uid: d3bd6e55-1c19-4a1f-84a0-db0f3b3c4cff
spec:
  clusterIP: None
  ports:
    - name: https
      port: 9100
      protocol: TCP
      targetPort: https
  selector:
    app: node-exporter
  sessionAffinity: None
  type: ClusterIP
daiqingliang commented 4 years ago

你的Docker镜像都下下来了吗,我自己手工下了十几个Docker镜像。 使用docker images 查看下下面两个镜像都有吗? dockerhub.qingcloud.com/coreos/kube-rbac-proxy:v0.4.1 dockerhub.qingcloud.com/prometheus/node-exporter:ks-v0.16.0

如果没有,那就是镜像问题,重新下载镜像: docker pull prom/node-exporter:ks-v0.16.0 docker tag prom/node-exporter:ks-v0.16.0 dockerhub.qingcloud.com/prometheus/node-exporter:ks-v0.16.0

docker pull carlosedp/kube-rbac-proxy:v0.4.1 docker tag carlosedp/kube-rbac-proxy:v0.4.1 dockerhub.qingcloud.com/prometheus/node-exporter:ks-v0.16.0

删除pod,重启服务 kubectl delete pod [pod-name] -n [namespace]

如果镜像有了,查看镜像版本是否匹配,我有手动拉Docker镜像导致版本匹配失败的情况。 如果还解决不了,我是没招了