[Perf] Linux/x64: 1 Regression on 8/29/2024 2:08:41 AM

[performanceautofiler[bot]]

Run Information

Name	Value
Architecture	x64
OS	ubuntu 22.04
Queue	ViperUbuntu
Baseline	1b5c48dc5958e20b4aa0f4cbfc21fddb8f81052c
Compare	2e7ee69fbcd2e653e2728cfb64e8a0faf9829e09
Diff	Diff
Configs	CompilationMode:tiered, RunKind:micro

Regressions in System.Numerics.Tests.Perf_BitOperations

Benchmark	Baseline	Test	Test/Base	Test Quality	Edge Detector	Baseline IR	Compare IR	IR Ratio
[TrailingZeroCount_ulong - Duration of single invocation](<https://pvscmdupload.z22.web.core.windows.net/reports/allTestHistory/refs/heads/main_x64_ubuntu 22.04/ViperUbuntu/System.Numerics.Tests.Perf_BitOperations.TrailingZeroCount_ulong.html>) 📝 - Benchmark Source ADX - Test Multi Config Graph	277.48 ns	367.18 ns	1.32	0.24	False

Test Report

Repro

General Docs link: https://github.com/dotnet/performance/blob/main/docs/benchmarking-workflow-dotnet-runtime.md

git clone https://github.com/dotnet/performance.git
python3 .\performance\scripts\benchmarks_ci.py -f net8.0 --filter 'System.Numerics.Tests.Perf_BitOperations*'

### System.Numerics.Tests.Perf_BitOperations.TrailingZeroCount_ulong #### ETL Files #### Histogram #### JIT Disasms ### Docs [Profiling workflow for dotnet/runtime repository](https://github.com/dotnet/performance/blob/master/docs/profiling-workflow-dotnet-runtime.md) [Benchmarking workflow for dotnet/runtime repository](https://github.com/dotnet/performance/blob/master/docs/benchmarking-workflow-dotnet-runtime.md)

[LoopedBard3]

Transfering this to start investigation into extreme increase in noisiness around July 19th. This noisiness increase seems to only be on the Viper machines and affects both Linux and Windows.

[LoopedBard3]

@AndyAyersMS @EgorBo

[EgorBo]

@EgorBot -amd -commit 39968e7 vs previous --filter System.Numerics.Tests.Perf_BitOperations.TrailingZeroCount_ulong

[EgorBo]

@EgorBot -amd -commit 39968e7 vs previous --filter System.Numerics.Tests.Perf_BitOperations.TrailingZeroCount_ulong

[dotnet-policy-service[bot]]

Tagging subscribers to this area: @JulieLeeMSFT, @jakobbotsch See info in area-owners.md if you want to be subscribed.

[EgorBo]

@EgorBot -amd -commit 39968e7 vs previous -profiler

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;
using System.Numerics;
using System.Runtime.CompilerServices;

public class Perf_BitOperations
{
    static T[] Array<T>(int count, int? seed = null)
    {
        var result = new T[count];
        var random = new Random(42);
        if (typeof(T) == typeof(byte) || typeof(T) == typeof(sbyte))
            random.NextBytes(Unsafe.As<byte[]>(result));
        else
            for (int i = 0; i < result.Length; i++)
                result[i] = GenerateValue<T>(random);
        return result;
    }

    static T GenerateValue<T>(Random random)
    {
        if (typeof(T) == typeof(uint))
            return (T)(object)(uint)random.Next();
        if (typeof(T) == typeof(ulong))
            return (T)(object)(ulong)random.Next();
        throw new NotImplementedException();
    }

    static uint[] input_uint = Array<uint>(1000);
    static ulong[] input_ulong = Array<ulong>(1000);

    [Benchmark]
    public int TrailingZeroCount_ulong()
    {
        int sum = 0;
        ulong[] input = input_ulong;
        for (int i = 0; i < input.Length; i++)
        {
            sum += BitOperations.TrailingZeroCount(input[i]);
        }
        return sum;
    }
}

[EgorBo]

@EgorBot -amd -commit 72f9ee0d26c23b3c58ec5af8aeea316095761646 vs 767e4165641583c73918168a920a10513000572c -profiler --envvars DOTNET_JitDisasm:TrailingZeroCount_ulong

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;
using System.Numerics;
using System.Runtime.CompilerServices;

public class Perf_BitOperations
{
    static T[] Array<T>(int count, int? seed = null)
    {
        var result = new T[count];
        var random = new Random(42);
        if (typeof(T) == typeof(byte) || typeof(T) == typeof(sbyte))
            random.NextBytes(Unsafe.As<byte[]>(result));
        else
            for (int i = 0; i < result.Length; i++)
                result[i] = GenerateValue<T>(random);
        return result;
    }

    static T GenerateValue<T>(Random random)
    {
        if (typeof(T) == typeof(uint))
            return (T)(object)(uint)random.Next();
        if (typeof(T) == typeof(ulong))
            return (T)(object)(ulong)random.Next();
        throw new NotImplementedException();
    }

    static uint[] input_uint = Array<uint>(1000);
    static ulong[] input_ulong = Array<ulong>(1000);

    [Benchmark]
    public int TrailingZeroCount_ulong()
    {
        int sum = 0;
        ulong[] input = input_ulong;
        for (int i = 0; i < input.Length; i++)
        {
            sum += BitOperations.TrailingZeroCount(input[i]);
        }
        return sum;
    }
}

[EgorBo]

@EgorBot -amd -commit 72f9ee0d26c23b3c58ec5af8aeea316095761646 vs previous -profiler --envvars DOTNET_JitDisasm:TrailingZeroCount_ulong

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;
using System.Numerics;
using System.Runtime.CompilerServices;

public class Perf_BitOperations
{
    static T[] Array<T>(int count, int? seed = null)
    {
        var result = new T[count];
        var random = new Random(42);
        if (typeof(T) == typeof(byte) || typeof(T) == typeof(sbyte))
            random.NextBytes(Unsafe.As<byte[]>(result));
        else
            for (int i = 0; i < result.Length; i++)
                result[i] = GenerateValue<T>(random);
        return result;
    }

    static T GenerateValue<T>(Random random)
    {
        if (typeof(T) == typeof(uint))
            return (T)(object)(uint)random.Next();
        if (typeof(T) == typeof(ulong))
            return (T)(object)(ulong)random.Next();
        throw new NotImplementedException();
    }

    static uint[] input_uint = Array<uint>(1000);
    static ulong[] input_ulong = Array<ulong>(1000);

    [Benchmark]
    public int TrailingZeroCount_ulong()
    {
        int sum = 0;
        ulong[] input = input_ulong;
        for (int i = 0; i < input.Length; i++)
        {
            sum += BitOperations.TrailingZeroCount(input[i]);
        }
        return sum;
    }
}

[EgorBo]

@EgorBot -amd -awsamd -commit 72f9ee0d26c23b3c58ec5af8aeea316095761646 vs previous -profiler --envvars DOTNET_JitDisasm:TrailingZeroCount_ulong

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;
using System.Numerics;
using System.Runtime.CompilerServices;

public class Perf_BitOperations
{
    static T[] Array<T>(int count, int? seed = null)
    {
        var result = new T[count];
        var random = new Random(42);
        if (typeof(T) == typeof(byte) || typeof(T) == typeof(sbyte))
            random.NextBytes(Unsafe.As<byte[]>(result));
        else
            for (int i = 0; i < result.Length; i++)
                result[i] = GenerateValue<T>(random);
        return result;
    }

    static T GenerateValue<T>(Random random)
    {
        if (typeof(T) == typeof(uint))
            return (T)(object)(uint)random.Next();
        if (typeof(T) == typeof(ulong))
            return (T)(object)(ulong)random.Next();
        throw new NotImplementedException();
    }

    static uint[] input_uint = Array<uint>(1000);
    static ulong[] input_ulong = Array<ulong>(1000);

    [Benchmark]
    public int TrailingZeroCount_ulong()
    {
        int sum = 0;
        ulong[] input = input_ulong;
        for (int i = 0; i < input.Length; i++)
        {
            sum += BitOperations.TrailingZeroCount(input[i]);
        }
        return sum;
    }
}

[EgorBo]

@EgorBot -awsamd -commit 72f9ee0d26c23b3c58ec5af8aeea316095761646 vs 767e4165641583c73918168a920a10513000572c -profiler --envvars DOTNET_JitDisasm:TrailingZeroCount_ulong

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;
using System.Numerics;
using System.Runtime.CompilerServices;

public class Perf_BitOperations
{
    static T[] Array<T>(int count, int? seed = null)
    {
        var result = new T[count];
        var random = new Random(42);
        if (typeof(T) == typeof(byte) || typeof(T) == typeof(sbyte))
            random.NextBytes(Unsafe.As<byte[]>(result));
        else
            for (int i = 0; i < result.Length; i++)
                result[i] = GenerateValue<T>(random);
        return result;
    }

    static T GenerateValue<T>(Random random)
    {
        if (typeof(T) == typeof(uint))
            return (T)(object)(uint)random.Next();
        if (typeof(T) == typeof(ulong))
            return (T)(object)(ulong)random.Next();
        throw new NotImplementedException();
    }

    static uint[] input_uint = Array<uint>(1000);
    static ulong[] input_ulong = Array<ulong>(1000);

    [Benchmark]
    public int TrailingZeroCount_ulong()
    {
        int sum = 0;
        ulong[] input = input_ulong;
        for (int i = 0; i < input.Length; i++)
        {
            sum += BitOperations.TrailingZeroCount(input[i]);
        }
        return sum;
    }
}

[EgorBo]

I think it is still interesting to investigate the source of randomness here, but looks like it is just one machine (I was not able to reproduce the same noise on AWS AMD machine) and one benchmark, moving to 10.0