[MeshMetric] Broken prometheus histograms

[tolwi]

What happened?

Policy

apiVersion: kuma.io/v1alpha1
kind: MeshMetric
metadata:
  name: ...
  namespace: kuma-system
spec:
  targetRef:
    kind: MeshService
    name: ...
  default:
    sidecar:
      profiles:
        appendProfiles:
          - name: Basic
    backends:
      - type: Prometheus
        prometheus:
          port: 5670
          path: /metrics

• envoy metrics (:9901/stats/prometheus)

  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="0.5"} 0
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="1"} 0
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="5"} 0
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="10"} 0
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="25"} 0
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="50"} 0
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="100"} 0
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="250"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="500"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="1000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="2500"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="5000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="10000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="30000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="60000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="300000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="600000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="1800000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="3600000"} 1
  envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name="other-service",le="+Inf"} 1
  envoy_cluster_upstream_rq_time_sum{envoy_cluster_name="other-service"} 155
  envoy_cluster_upstream_rq_time_count{envoy_cluster_name="other-service"} 1

• dp metrics (:5670/metrics)

  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="0.5"} 0
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="1"} 0
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="5"} 0
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="10"} 0
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="25"} 0
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="50"} 0
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="100"} 0
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="250"} 1
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="500"} 2
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="1000"} 3
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="2500"} 4
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="5000"} 5
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="10000"} 6
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="30000"} 7
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="60000"} 8
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="300000"} 9
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="600000"} 10
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="1.8e+06"} 11
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="3.6e+06"} 12
  envoy_cluster_upstream_rq_time_bucket{.. envoy_cluster_name="other-service",le="+Inf"} 1
  envoy_cluster_upstream_rq_time_sum{.. envoy_cluster_name="other-service"} 155
  envoy_cluster_upstream_rq_time_count{.. envoy_cluster_name="other-service"} 1

After digging for a while, I believe I've found the root cause: there is an inconsistency in histogram types between Prometheus and Otel. Envoy exposes a 'cumulative' histogram, which is then fetched into an Otel histogram, considered 'native'. Therefore, when this histogram is exported to Prometheus, it should be converted back into a 'cumulative' format."(here)

[tolwi]

how to reproduce

```golang package metrics import ( "context" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/stretchr/testify/assert" otel_prom "go.opentelemetry.io/otel/exporters/prometheus" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" "net/http/httptest" "strings" "testing" "time" ) const InHistogram = ` # TYPE envoy_cluster_upstream_cx_connect_ms histogram envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="0.5"} 0 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="1"} 0 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="5"} 0 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="10"} 0 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="25"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="50"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="100"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="250"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="500"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="1000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="2500"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="5000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="10000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="30000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="60000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="300000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="600000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="1800000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="3600000"} 1 envoy_cluster_upstream_cx_connect_ms_bucket{envoy_cluster_name="ads_cluster",le="+Inf"} 1 envoy_cluster_upstream_cx_connect_ms_sum{envoy_cluster_name="ads_cluster"} 20.5 envoy_cluster_upstream_cx_connect_ms_count{envoy_cluster_name="ads_cluster"} 1 ` type testProducer struct { metrics metricdata.ScopeMetrics } func (p *testProducer) Produce(ctx context.Context) ([]metricdata.ScopeMetrics, error) { return []metricdata.ScopeMetrics{p.metrics}, nil } func Test(t *testing.T) { metrics, err := AggregatedOtelMutator()(strings.NewReader(InHistogram)) assert.NoError(t, err) p := &testProducer{ metrics: metricdata.ScopeMetrics{ Metrics: FromPrometheusMetrics(metrics, "", "", "", make(map[string]string), time.Now()), }, } promExporter, err := otel_prom.New(otel_prom.WithProducer(p)) assert.NoError(t, err) sdkmetric.NewMeterProvider(sdkmetric.WithReader(promExporter)) handler := promhttp.Handler() recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, httptest.NewRequest("GET", "/metrics", nil)) println(string(recorder.Body.Bytes())) } ```