[bug] Dependencies are not sampled

[ep-workai]

Component

OpenTelemetry.Instrumentation.SqlClient

Package Version

Runtime Version

net472, net8.0

Description

After sampler attached seems normal traces are sampled, but not dependencies ones for example from SqlClient library. There are a lot of dependency traces in storage.

Steps to Reproduce

1. Add SqlClient instrumentation package
2. Add traces
3. Add SQL calls
4. Observe stored traces

Expected Result

Dependencies should be also sampled

Actual Result

Dependencies are not sampled

Additional Context

No response

Example code with configuration:

using Azure.Core;
using Azure.Core.Pipeline;
using Azure.Monitor.OpenTelemetry.Exporter;
using Microsoft.AspNetCore.Http;
using Microsoft.Data.SqlClient;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using OpenTelemetry;
using OpenTelemetry.Logs;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using company.Logging;
using company.Logging.Filters;
using company.Logging.Options;
using company.Logging.Processors;
using company.Shared.Configuration;

namespace company.Shared.Extensions;

public static partial class IcompanyAppBuilderExtensions
{
    [KeepReference]
    private static TracerProvider _tracerProvider;

    private static void AddOpenTelemetry(IcompanyAppBuilder @this,
        (string Name, Version Version, string InstanceName, string Namespace) serviceInfo)
    {
        var options = @this.Configuration.GetcompanyOptions<LoggingOptions>();

        _tracerProvider = CreateTraceProvider(@this, serviceInfo, options.OpenTelemetry.Sampling);

        OpenTelemetryBuilder openTelemetryBuilder = @this.Services.AddOpenTelemetry()
            .ConfigureResource(
                resource => resource.Clear()
                    .AddService(
                        serviceInfo.Name,
                        serviceInfo.Namespace,
                        serviceInfo.Version.ToString(),
                        false,
                        serviceInfo.InstanceName));

        if (options.OpenTelemetry.MetricsEnabled)
        {
            openTelemetryBuilder.WithMetrics(
                metrics => metrics
                    .AddMeter(serviceInfo.Name)
                    .AddHttpClientInstrumentation()
                    .AddAspNetCoreInstrumentation()
                    .AddConsoleExporter()
                    .AddAzureMonitorMetricExporter(
                        x => ConfigureAzureMonitorExporter(x, @this.Configuration.GetcompanyOptions<LoggingOptions>())));
        }

        AddDebugLogProcessor(@this, out var debugLogProcessor);

        AddLogs(@this, serviceInfo, debugLogProcessor);
    }

    private static void AddLogs(IcompanyAppBuilder @this,
        (string Name, Version Version, string InstanceName, string Namespace) serviceInfo,
        DebugLogProcessor? debugLogProcessor)
    {
        var loggingOptions = @this.Configuration.GetcompanyOptions<LoggingOptions>();

        @this.LoggingBuilder.AddOpenTelemetry(
                options =>
                {
                    options.IncludeScopes = true;
                    options.IncludeFormattedMessage = true;
                    options.ParseStateValues = true;

                    options
                        .SetResourceBuilder(
                            ResourceBuilder.CreateEmpty()
                                .AddService(
                                    serviceInfo.Name,
                                    serviceInfo.Namespace,
                                    serviceInfo.Version.ToString(),
                                    false,
                                    serviceInfo.InstanceName))
                        .AddProcessor(new SensitiveDataEraserProcessor());

                    if (@this.Services.Any(
                            serviceDescriptor => serviceDescriptor.ServiceType == typeof(DebugLogProcessor)))
                    {
                        options.AddProcessor(debugLogProcessor);
                    }

                    options
                        .AddConsoleExporter()
                        .AddAzureMonitorLogExporter(
                            y => ConfigureAzureMonitorExporter(y, loggingOptions));
                })
            .AddFilter(new GlobalLogFilter().Filter)
            .SetMinimumLevel(loggingOptions.MinLogLevel);
    }

    private static TracerProvider CreateTraceProvider(IcompanyAppBuilder companyAppBuilder,
        (string Name, Version Version, string InstanceName, string Namespace) serviceInfo,
        double? sampling)
    {
        var loggingOptions = companyAppBuilder.Configuration.GetcompanyOptions<LoggingOptions>();

        AddDebugTraceProcessor(companyAppBuilder, out var debugTraceProcessor);

        TracerProviderBuilder tracing = Sdk.CreateTracerProviderBuilder();

        tracing.AddSource([serviceInfo.Name, .. companyAppBuilder.ActivitySources]);
        tracing.AddProcessor<TailSamplingProcessor>();

        if (companyAppBuilder.Services.Any(
                serviceDescriptor => serviceDescriptor.ServiceType == typeof(DebugTraceProcessor)))
        {
            tracing.AddProcessor(debugTraceProcessor);
        }

        tracing
            .AddAspNetCoreInstrumentation(
                opt =>
                {
                    opt.RecordException = true;
                    opt.Filter = AspNetCoreFilter;
                })
            .AddHttpClientInstrumentation(x => x.RecordException = true)
            .AddEntityFrameworkCoreInstrumentation()
            .AddSqlClientInstrumentation(
                x =>
                {
                    x.Filter = SqlFilter;
                    x.RecordException = true;
                    x.SetDbStatementForText = loggingOptions.OpenTelemetry.SqlClient?.StatementsVisible ?? false;
                    x.SetDbStatementForStoredProcedure = loggingOptions.OpenTelemetry.SqlClient?.StatementsVisible ?? false;
                })
            .AddHangfireInstrumentation(x => x.RecordException = true)
            .SetErrorStatusOnException()
            .AddConsoleExporter()
            .AddAzureMonitorTraceExporter(
                x => ConfigureAzureMonitorExporter(x, loggingOptions));

        // it's important to put this statement after adding all others which can override sampler
        tracing.SetSampler(new ParentBasedRecordSampler(sampling));

        return tracing.Build();
    }

    private static bool SqlFilter(object arg)
    {
        var command = (SqlCommand)arg;

        // ignore SQL internal procedures
        return !command.CommandText.StartsWith("sp_", StringComparison.OrdinalIgnoreCase);
    }

    private static void ConfigureAzureMonitorExporter(AzureMonitorExporterOptions exporterOptions,
        LoggingOptions loggingOptions)
    {
        exporterOptions.Diagnostics.IsDistributedTracingEnabled = true;

        exporterOptions.ConnectionString =
            loggingOptions.OpenTelemetry.AzureMonitorConnectionString;

        exporterOptions.RetryPolicy = new RetryPolicy(
            6,
            DelayStrategy.CreateExponentialDelayStrategy(
                TimeSpan.FromMilliseconds(50),
                TimeSpan.FromSeconds(30)));
    }

    private static void AddDebugLogProcessor(IcompanyAppBuilder @this,
        out DebugLogProcessor? debugLogProcessor)
    {
        if (@this.Environment.IsDevelopment())
        {
            debugLogProcessor = new DebugLogProcessor();
            @this.Services.AddSingleton(debugLogProcessor);
        }
        else
        {
            debugLogProcessor = null;
        }
    }

    private static void AddDebugTraceProcessor(IcompanyAppBuilder @this,
        out DebugTraceProcessor? debugTraceProcessor)
    {
        if (@this.Environment.IsDevelopment())
        {
            debugTraceProcessor = new DebugTraceProcessor();
            @this.Services.AddSingleton(debugTraceProcessor);
        }
        else
        {
            debugTraceProcessor = null;
        }
    }

    private static bool AspNetCoreFilter(HttpContext context)
    {
        if (!context.Request.Path.HasValue || context.Request.Path == PathString.Empty)
        {
            return true;
        }

        // ignore health-checks
        if (context.Request.Path.Value.Contains($"/{HealthCheckConfiguration.UrlPath}/"))
        {
            return false;
        }

        // ignore signalr hub requests
        if (((string[]) ["signalr", "notificationsHub", "socialHub"]).Any(
                x => context.Request.Path.Value.Contains(x)))
        {
            return false;
        }

        return true;
    }
}

[cijothomas]

What are "dependencies"? Are you referring to the Database Spans? Please share a minimal repro explaining the issue, so we can help further.

[ep-workai]

In context of Azure logging storage (Azure Log Analytics) Dependencies are traces of special category for external activities. Database spans are indeed in this category, also HTTP calls, etc. with separate instrumentation packages.

I provided an example in edited post version.

[cijothomas]

@ep-workai Please share a minimal repro explaining the issue, so we can help further. You can take the helloworld OpenTelemetry example, and incrementally add things to until the issue reproduces. It is hard for me to help otherwise.