search

timescale / prometheus-postgresql-adapter

Use PostgreSQL as a remote storage database for Prometheus
Apache License 2.0
335 stars 66 forks source link

Too many metrics data are logged in pg-adapter container log #117

Open suridaddy opened 3 years ago

suridaddy commented 3 years ago

image version

prometheus-postgresql-adapter:0.4.1

description

following metrics data are print out in pg-adapter container log, 

scrape_samples_scraped{endpoint="https-metrics",instance="9.181.65.57:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="worker14.bjdemo.eti.cdl.ibm.com",service="kubelet"} 6928 1614577680918
kubelet_volume_stats_capacity_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="data-rabbitmq-rabbitmq-ha-0",service="kubelet"} 1089312129024 1614577681676
container_cpu_usage_seconds_total{container="keycloak",cpu="total",endpoint="https-metrics",id="/kubepods.slice/kubepods-pode0081be9_be1d_42b2_a8be_492352c219e9.slice/crio-31d553855390f5884ad3502c0c7101aa61ae363ed281eae4136785326b4d2f4c.scope",image="image-registry.openshift-image-registry.svc:5000/eco-common/keycloak:12.0.2",instance="9.181.65.50:10250",job="kubelet",metrics_path="/metrics/cadvisor",name="k8s_keycloak_keycloak-2_eco-common_e0081be9-be1d-42b2-a8be-492352c219e9_0",namespace="eco-common",node="worker07.bjdemo.eti.cdl.ibm.com",pod="keycloak-2",service="kubelet"} 16311.074252689 1614577675794
scrape_samples_scraped{endpoint="https-metrics",instance="9.181.65.72:10250",job="kubelet",metrics_path="/metrics/probes",namespace="kube-system",node="worker29.bjdemo.eti.cdl.ibm.com",service="kubelet"} 4 1614577682287
scrape_duration_seconds{container="kube-rbac-proxy",endpoint="https",instance="worker21.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-qds87",service="node-exporter"} 0.055883897 1614577682450
ALERTS{alertname="failedJobs",alertstate="firing",condition="true",container="kube-rbac-proxy-main",endpoint="https-main",instance="10.131.0.22:8443",job="kube-state-metrics",job_name="curator-1614310200",namespace="openshift-logging",pod="kube-state-metrics-74886775d5-ccxp6",service="kube-state-metrics"} 1 1614577682201
ALERTS_FOR_STATE{alertname="failedJobs",condition="true",container="kube-rbac-proxy-main",endpoint="https-main",instance="10.131.0.22:8443",job="kube-state-metrics",job_name="minimum-a7s-cj-retention-1614389400",namespace="apic",pod="kube-state-metrics-74886775d5-ccxp6",service="kube-state-metrics"} 1614562709 1614577682201
scrape_samples_post_metric_relabeling{endpoint="https-metrics",instance="9.181.65.72:10250",job="kubelet",metrics_path="/metrics",namespace="kube-system",node="worker29.bjdemo.eti.cdl.ibm.com",service="kubelet"} 0 1614577682964
mongodb_mongod_replset_member_health{instance="mongodb-common-primary-0.mongodb-common-headless.eco-common.svc.cluster.local:9216",job="mongodbprimary",name="mongodb-common-secondary-0.mongodb-common-headless.eco-common.svc.cluster.local:27017",set="rs0",state="SECONDARY"} 1 1614577683737
kubelet_volume_stats_available_bytes{endpoint="https-metrics",instance="9.181.65.47:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker04.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="data-ldapha-ldap-ha-2",service="kubelet"} 787179634688 1614577683969
kubelet_volume_stats_capacity_bytes{endpoint="https-metrics",instance="9.181.65.47:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker04.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="export-minio-0",service="kubelet"} 1089312129024 1614577683969
up{container="kube-rbac-proxy",endpoint="https",instance="master01.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-n49r2",service="node-exporter"} 1 1614577683985
scrape_samples_post_metric_relabeling{instance="mongodb-common-secondary-0.mongodb-common-headless.eco-common.svc.cluster.local:9216",job="mongodbsecondary0"} 8 1614577684655
scrape_samples_post_metric_relabeling{endpoint="https-metrics",instance="9.181.65.42:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="master02.bjdemo.eti.cdl.ibm.com",service="kubelet"} 0 1614577679357
kubelet_volume_stats_available_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="datadir-mongodb-common-secondary-0",service="kubelet"} 787076874240 1614577681676
kubelet_volume_stats_capacity_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="pg-pvc",service="kubelet"} 1089312129024 1614577681676
scrape_samples_scraped{container="kube-rbac-proxy",endpoint="https",instance="worker13.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-8t8dx",service="node-exporter"} 1343 1614577681996
container_memory_working_set_bytes{endpoint="https-metrics",id="/kubepods.slice/kubepods-pode0081be9_be1d_42b2_a8be_492352c219e9.slice",instance="9.181.65.50:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="eco-common",node="worker07.bjdemo.eti.cdl.ibm.com",pod="keycloak-2",service="kubelet"} 660758528 1614577667437
ALERTS{alertname="failedJobs",alertstate="firing",condition="true",container="kube-rbac-proxy-main",endpoint="https-main",instance="10.131.0.22:8443",job="kube-state-metrics",job_name="minimum-a7s-cj-retention-1614562200",namespace="apic",pod="kube-state-metrics-74886775d5-ccxp6",service="kube-state-metrics"} 1 1614577682201
up{endpoint="https-metrics",instance="9.181.65.65:10250",job="kubelet",metrics_path="/metrics",namespace="kube-system",node="worker22.bjdemo.eti.cdl.ibm.com",service="kubelet"} 1 1614577682658
scrape_samples_post_metric_relabeling{container="kube-rbac-proxy",endpoint="https",instance="worker05.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-h4grd",service="node-exporter"} 0 1614577682336
scrape_duration_seconds{endpoint="https-metrics",instance="9.181.65.62:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="worker19.bjdemo.eti.cdl.ibm.com",service="kubelet"} 0.315437529 1614577682776
up{instance="mongodb-common-primary-0.mongodb-common-headless.eco-common.svc.cluster.local:9216",job="mongodbprimary"} 1 1614577683737
kubelet_volume_stats_available_bytes{endpoint="https-metrics",instance="9.181.65.47:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker04.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="eco-helm-repo-chartmuseum",service="kubelet"} 787285540864 1614577683969
scrape_samples_post_metric_relabeling{endpoint="https-metrics",instance="9.181.65.47:10250",job="kubelet",metrics_path="/metrics",namespace="kube-system",node="worker04.bjdemo.eti.cdl.ibm.com",service="kubelet"} 18 1614577683969
mongodb_exporter_last_scrape_error{instance="mongodb-common-secondary-0.mongodb-common-headless.eco-common.svc.cluster.local:9216",job="mongodbsecondary0"} 0 1614577684655
kubelet_volume_stats_available_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="export-minio-2",service="kubelet"} 787178586112 1614577681676
kubelet_volume_stats_capacity_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="marketplace",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="oss-backend",service="kubelet"} 1089312129024 1614577681676
container_network_receive_bytes_total{container="POD",endpoint="https-metrics",id="/kubepods.slice/kubepods-pode0081be9_be1d_42b2_a8be_492352c219e9.slice/crio-79b2aaa8aae69b2bca79bd25face70e253b570ed6bf90033b8bd66467dd17b23.scope",image="quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba47753b76f9f77e7ff4659dff95df3cbe347895aba4288e089fb978bfdc8a6e",instance="9.181.65.50:10250",interface="eth0",job="kubelet",metrics_path="/metrics/cadvisor",name="k8s_POD_keycloak-2_eco-common_e0081be9-be1d-42b2-a8be-492352c219e9_0",namespace="eco-common",node="worker07.bjdemo.eti.cdl.ibm.com",pod="keycloak-2",service="kubelet"} 38352301749 1614577676881
ALERTS{alertname="failedJobs",alertstate="firing",condition="true",container="kube-rbac-proxy-main",endpoint="https-main",instance="10.131.0.22:8443",job="kube-state-metrics",job_name="elasticsearch-rollover-infra-1612021500",namespace="openshift-logging",pod="kube-state-metrics-74886775d5-ccxp6",service="kube-state-metrics"} 1 1614577682201
scrape_samples_post_metric_relabeling{endpoint="https-metrics",instance="9.181.65.65:10250",job="kubelet",metrics_path="/metrics",namespace="kube-system",node="worker22.bjdemo.eti.cdl.ibm.com",service="kubelet"} 0 1614577682658
up{endpoint="https-metrics",instance="9.181.65.56:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="worker13.bjdemo.eti.cdl.ibm.com",service="kubelet"} 1 1614577682851
scrape_samples_post_metric_relabeling{instance="mongodb-common-primary-0.mongodb-common-headless.eco-common.svc.cluster.local:9216",job="mongodbprimary"} 8 1614577683737
scrape_samples_scraped{container="kube-rbac-proxy",endpoint="https",instance="master03.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-2xbtp",service="node-exporter"} 2409 1614577683698
mongodb_mongod_replset_member_health{instance="mongodb-common-secondary-0.mongodb-common-headless.eco-common.svc.cluster.local:9216",job="mongodbsecondary0",name="mongodb-common-secondary-1.mongodb-common-headless.eco-common.svc.cluster.local:27017",set="rs0",state="SECONDARY"} 1 1614577684655
scrape_samples_scraped{endpoint="https-metrics",instance="9.181.65.50:10250",job="kubelet",metrics_path="/metrics/probes",namespace="kube-system",node="worker07.bjdemo.eti.cdl.ibm.com",service="kubelet"} 81 1614577685256
scrape_samples_scraped{endpoint="https-metrics",instance="9.181.65.42:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="master02.bjdemo.eti.cdl.ibm.com",service="kubelet"} 10803 1614577679357
kubelet_volume_stats_available_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="data-redis-redis-ha-server-0",service="kubelet"} 787084214272 1614577681676
kubelet_volume_stats_capacity_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="export-minio-2",service="kubelet"} 1089312129024 1614577681676
scrape_duration_seconds{container="kube-rbac-proxy",endpoint="https",instance="worker13.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-8t8dx",service="node-exporter"} 0.101616022 1614577681996
container_memory_usage_bytes{container="keycloak",endpoint="https-metrics",id="/kubepods.slice/kubepods-pode0081be9_be1d_42b2_a8be_492352c219e9.slice/crio-31d553855390f5884ad3502c0c7101aa61ae363ed281eae4136785326b4d2f4c.scope",image="image-registry.openshift-image-registry.svc:5000/eco-common/keycloak:12.0.2",instance="9.181.65.50:10250",job="kubelet",metrics_path="/metrics/cadvisor",name="k8s_keycloak_keycloak-2_eco-common_e0081be9-be1d-42b2-a8be-492352c219e9_0",namespace="eco-common",node="worker07.bjdemo.eti.cdl.ibm.com",pod="keycloak-2",service="kubelet"} 648445952 1614577675794
scrape_samples_post_metric_relabeling{endpoint="https-metrics",instance="9.181.65.68:10250",job="kubelet",metrics_path="/metrics",namespace="kube-system",node="worker25.bjdemo.eti.cdl.ibm.com",service="kubelet"} 0 1614577682395
ALERTS_FOR_STATE{alertname="failedJobs",condition="true",container="kube-rbac-proxy-main",endpoint="https-main",instance="10.131.0.22:8443",job="kube-state-metrics",job_name="elasticsearch-im-infra-1614573000",namespace="openshift-logging",pod="kube-state-metrics-74886775d5-ccxp6",service="kube-state-metrics"} 1614573156 1614577682201
ALERTS_FOR_STATE{alertname="failedJobs",condition="true",container="kube-rbac-proxy-main",endpoint="https-main",instance="10.131.0.22:8443",job="kube-state-metrics",job_name="minimum-a7s-cj-retention-1614475800",namespace="apic",pod="kube-state-metrics-74886775d5-ccxp6",service="kube-state-metrics"} 1614562709 1614577682201
scrape_samples_scraped{container="kube-rbac-proxy",endpoint="https",instance="worker05.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-h4grd",service="node-exporter"} 7750 1614577682336
up{endpoint="https-metrics",instance="9.181.65.62:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="worker19.bjdemo.eti.cdl.ibm.com",service="kubelet"} 1 1614577682776
mongodb_mongod_replset_number_of_members{instance="mongodb-common-primary-0.mongodb-common-headless.eco-common.svc.cluster.local:9216",job="mongodbprimary",set="rs0"} 3 1614577683737
kubelet_volume_stats_available_bytes{endpoint="https-metrics",instance="9.181.65.47:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker04.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="eco-grafana",service="kubelet"} 787039125504 1614577683969
scrape_samples_scraped{endpoint="https-metrics",instance="9.181.65.47:10250",job="kubelet",metrics_path="/metrics",namespace="kube-system",node="worker04.bjdemo.eti.cdl.ibm.com",service="kubelet"} 1240 1614577683969
scrape_duration_seconds{endpoint="https-metrics",instance="9.181.65.69:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="worker26.bjdemo.eti.cdl.ibm.com",service="kubelet"} 0.344350686 1614577680891
kubelet_volume_stats_available_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="marketplace",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="marketplace-backend",service="kubelet"} 787216334848 1614577681676
scrape_duration_seconds{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="kube-system",node="worker06.bjdemo.eti.cdl.ibm.com",service="kubelet"} 0.128174008 1614577681676
scrape_duration_seconds{endpoint="https-metrics",instance="9.181.65.50:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="worker07.bjdemo.eti.cdl.ibm.com",service="kubelet"} 1.080274174 1614577681128
up{container="kube-rbac-proxy",endpoint="https",instance="worker28.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-djs67",service="node-exporter"} 1 1614577682392
scrape_samples_scraped{endpoint="https-metrics",instance="9.181.65.48:10250",job="kubelet",metrics_path="/metrics/probes",namespace="kube-system",node="worker05.bjdemo.eti.cdl.ibm.com",service="kubelet"} 29 1614577682923
scrape_samples_post_metric_relabeling{endpoint="https-metrics",instance="9.181.65.56:10250",job="kubelet",metrics_path="/metrics/cadvisor",namespace="kube-system",node="worker13.bjdemo.eti.cdl.ibm.com",service="kubelet"} 0 1614577682851
kubelet_volume_stats_capacity_bytes{endpoint="https-metrics",instance="9.181.65.47:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker04.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="data-rabbitmq-rabbitmq-ha-2",service="kubelet"} 1089312129024 1614577683969
up{container="kube-rbac-proxy",endpoint="https",instance="worker26.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-55xfm",service="node-exporter"} 1 1614577684248
mongodb_mongod_replset_member_state{instance="mongodb-common-secondary-0.mongodb-common-headless.eco-common.svc.cluster.local:9216",job="mongodbsecondary0",name="mongodb-common-secondary-1.mongodb-common-headless.eco-common.svc.cluster.local:27017",set="rs0",state="SECONDARY"} 2 1614577684655
kubelet_volume_stats_available_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="eco-grafana",service="kubelet"} 787245694976 1614577681676
kubelet_volume_stats_capacity_bytes{endpoint="https-metrics",instance="9.181.65.49:10250",job="kubelet",metrics_path="/metrics",namespace="eco-common",node="worker06.bjdemo.eti.cdl.ibm.com",persistentvolumeclaim="retention-pvc",service="kubelet"} 1089312129024 1614577681676
scrape_samples_post_metric_relabeling{container="kube-rbac-proxy",endpoint="https",instance="worker13.bjdemo.eti.cdl.ibm.com",job="node-exporter",namespace="openshift-monitoring",pod="node-exporter-8t8dx",service="node-exporter"} 0 1614577681996
....

question

too many logs generated by pg-adapter as above, do we have any way to stop the output?

Harkishen-Singh commented 3 years ago

Hey @suridaddy, this repository is slate as mentioned in the README.md. The actual production code/remote-storage lives here. Please consider using that and opening the issue in that repository, if you face any problem or reach out to us in #promscale channel in the timescale slack.