open-telemetry / opentelemetry-go

OpenTelemetry Go API and SDK
https://opentelemetry.io/docs/languages/go
Apache License 2.0
5.28k stars 1.08k forks source link

Combining `otlp` exporter with `NewWithExactDistribution` and `ValueRecorder` produces gauge in prometheus metrics #1790

Closed imle closed 2 years ago

imle commented 3 years ago

Description

From my understanding of the docs that I could find on this, NewWithExactDistribution should produce a large amount of data when looking at the prometheus metrics endpoint on the opentelemetry-collector. Instead I get a gauge.

When using:

simple.NewWithHistogramDistribution(
    histogram.WithExplicitBoundaries([]float64{
        0.001, 0.01, 0.1, 1, 10, 100, 1000,
    }),
)

I get the expected histogram output in the /metrics endpoint of the collector.

When using:

simple.NewWithExactDistribution()

I get a gauge instead. If this is expected behavior then it does not seem documented anywhere yet (understandable given the state of the API). However, from my understanding of the function's documentation, I should see histogram with possibly infinite buckets?

Environment

Steps To Reproduce

package main

import (
    "context"
    "fmt"
    "log"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp"
    "go.opentelemetry.io/otel/exporters/otlp/otlpgrpc"
    "go.opentelemetry.io/otel/metric"
    "go.opentelemetry.io/otel/metric/global"
    metric2 "go.opentelemetry.io/otel/sdk/export/metric"
    "go.opentelemetry.io/otel/sdk/metric/aggregator/histogram"
    controller "go.opentelemetry.io/otel/sdk/metric/controller/basic"
    processor "go.opentelemetry.io/otel/sdk/metric/processor/basic"
    "go.opentelemetry.io/otel/sdk/metric/selector/simple"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    "go.opentelemetry.io/otel/trace"
)

// Change this to see the different behaviors.
const UseHistogramAggregatorSelector = true

func main() {
    ctx := context.Background()

    driver := otlpgrpc.NewDriver(
        otlpgrpc.WithInsecure(),
        otlpgrpc.WithEndpoint("localhost:4317"),
    )

    exp, err := otlp.NewExporter(ctx, driver)
    if err != nil {
        log.Fatalf("failed to create the collector exporter: %v", err)
    }
    defer func() {
        ctx, cancel := context.WithTimeout(ctx, time.Second)
        defer cancel()
        if err := exp.Shutdown(ctx); err != nil {
            otel.Handle(err)
        }
    }()

    tp := sdktrace.NewTracerProvider(
        sdktrace.WithSampler(sdktrace.AlwaysSample()),
        sdktrace.WithBatcher(
            exp,
            sdktrace.WithBatchTimeout(5*time.Second),
            sdktrace.WithMaxExportBatchSize(10),
        ),
    )
    defer func() {
        ctx, cancel := context.WithTimeout(ctx, time.Second)
        defer cancel()
        if err := tp.Shutdown(ctx); err != nil {
            otel.Handle(err)
        }
    }()
    otel.SetTracerProvider(tp)

    var agg metric2.AggregatorSelector
    if UseHistogramAggregatorSelector {
        agg = simple.NewWithHistogramDistribution(
            histogram.WithExplicitBoundaries([]float64{
                0.001, 0.01, 0.1, 1, 10, 100, 1000,
            }),
        )
    } else {
        agg = simple.NewWithExactDistribution()
    }

    pusher := controller.New(
        processor.New(agg, exp),
        controller.WithExporter(exp),
        controller.WithCollectPeriod(2*time.Second),
    )
    global.SetMeterProvider(pusher.MeterProvider())

    if err := pusher.Start(ctx); err != nil {
        log.Fatalf("could not start metric controoler: %v", err)
    }
    defer func() {
        ctx, cancel := context.WithTimeout(ctx, time.Second)
        defer cancel()
        if err := pusher.Stop(ctx); err != nil {
            otel.Handle(err)
        }
    }()

    tracer := otel.Tracer("test-tracer")
    meter := global.Meter("test-meter")

    var name string
    if UseHistogramAggregatorSelector {
        name = "an_important_metric_hist"
    } else {
        name = "an_important_metric_gauge"
    }

    counter := metric.Must(meter).
        NewFloat64ValueRecorder(name,
            metric.WithDescription("Measures the cumulative epicness of the app"),
        )

    ctx, span := tracer.Start(
        ctx,
        "DifferentCollectors-Example")
    defer span.End()
    for i := 0; i < 10; i++ {
        work(ctx, i, tracer, counter)
    }

    log.Printf("Done!")
}

func work(ctx context.Context, i int, tracer trace.Tracer, counter metric.Float64ValueRecorder) {
    st := time.Now()
    _, iSpan := tracer.Start(ctx, fmt.Sprintf("Sample-%d", i))

    log.Printf("Doing really hard work (%d / 10)\n", i+1)

    <-time.After(time.Second)
    iSpan.End()
    counter.Record(ctx, time.Since(st).Seconds())
}

opentelemetry-collector config

receivers:
  # Make sure to add the otlp receiver.
  # This will open up the receiver on port 4317
  otlp:
    protocols:
      grpc:
        endpoint: "0.0.0.0:4317"
processors:
extensions:
  health_check: {}
exporters:
  jaeger:
    endpoint: "jaeger:14250"
    insecure: true
  prometheus:
    endpoint: 0.0.0.0:8889
    namespace: "paperfree"
  logging:

service:
  extensions: [health_check]
  pipelines:
    traces:
      receivers: [otlp]
      processors: []
      exporters: [jaeger]

    metrics:
      receivers: [otlp]
      processors: []
      exporters: [prometheus, logging]

Expected behavior

I would expect a histogram or at least an additive output.

MrAlias commented 2 years ago

Closing, stale. The new SDK merged in https://github.com/open-telemetry/opentelemetry-go/pull/3175 should allow the needed configuration of aggregation output. If not, please open a new issue describing the failure.