Solve MinIterationTime warning

[swtrse]

Hello,

I have a Benchmark that looks like

namespace Cobra.Exoda.Matching.Benchmarks.Base;

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Jobs;
using JetBrains.Annotations;

[CategoriesColumn]
[MinColumn]
[MaxColumn]
[MemoryDiagnoser(false)]
[HideColumns("Job", "Type")]
[ShortRunJob(RuntimeMoniker.Net80)]
[ShortRunJob(RuntimeMoniker.Net90)]
//[ShortRunJob(RuntimeMoniker.NativeAot80)] // Bug
//[ShortRunJob(RuntimeMoniker.NativeAot90)] // Bug
public class BenchmarkBase
{
    [Params(0, 10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000, 100_000_000)]
    public int Prefill { get; [UsedImplicitly] set; }

    [Params(10)]
    public int Inserts { get; [UsedImplicitly] set; }

    [Params(1_00, 2_000, 5_000, 10_000)] public int PriceRange { get; [UsedImplicitly] set; }
}

namespace Cobra.Exoda.Matching.Benchmarks;

using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.Numerics;
using Base;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNetVisualizer;
using JetBrains.Annotations;
using Nethermind.Int256;
using SortedDictionary.BigInteger;
using SortedDictionary.UInt256;
using Utils;

[PublicAPI]
[RichHtmlExporter("Benchmark of Collection Performance",
                  ["Prefill", "Inserts", "PriceRange"],
                  ["Mean", "Allocated"],
                  ["Mean", "Min", "Max", "Allocated"],
                  dividerMode: RenderTableDividerMode.SeparateTables,
                  htmlWrapMode: HtmlDocumentWrapMode.RichDataTables)]
[GenericTypeArguments(typeof(OrderDataBigIntegerKey), typeof(OrderDataBigInteger))]
[GenericTypeArguments(typeof(OrderDataUInt256Key), typeof(OrderDataUInt256))]
[Display(Name = "Benchmark of Flat Dictionary Item Insert for IDictionary<{0}, {1}>", GroupName = "Benchmark of Dictionaries Item Insert")]

// ReSharper disable once ClassCanBeSealed.Global
public class AddItemBenchmarkFlat<TKey, TData> : BenchmarkBase
    where TKey : notnull
    where TData : notnull
{
    [GlobalSetup]
    public void A_Global_Setup()
    {
        _orderDataKeys = new TKey[Inserts];
        _orderDataArray = new TData[Inserts];
        _prefillArray = new TData[Prefill];
        var rnd = new Random();
        if (typeof(TData) == typeof(OrderDataBigInteger))
        {
            Parallel.For(0,
                         Inserts,
                         i =>
                         {
                             var price = new BigInteger(rnd.NextInt64(0, PriceRange));
                             var orderId = new Guid(i + Prefill, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
                             _orderDataKeys[i] = (dynamic)new OrderDataBigIntegerKey(price, orderId);
                             _orderDataArray[i] = (dynamic)new OrderDataBigInteger(price, orderId, 1);
                         });
            Parallel.For(0,
                         Prefill,
                         i =>
                         {
                             var price = new BigInteger(rnd.NextInt64(0, PriceRange));
                             var orderId = new Guid(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
                             _prefillArray[i] = (dynamic)new OrderDataBigInteger(price, orderId, 1);
                         });
        }
        else if (typeof(TData) == typeof(OrderDataUInt256))
        {
            Parallel.For(0,
                         Inserts,
                         i =>
                         {
                             var price = new UInt256((ulong)rnd.NextInt64(0, PriceRange));
                             var orderId = new Guid(i + Prefill, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
                             _orderDataKeys[i] = (dynamic)new OrderDataUInt256Key(price, orderId);
                             _orderDataArray[i] = (dynamic)new OrderDataUInt256(price, orderId, 1);
                         });
            Parallel.For(0,
                         Prefill,
                         i =>
                         {
                             var price = new UInt256((ulong)rnd.NextInt64(0, PriceRange));
                             var orderId = new Guid(i, 0, 0, [0, 0, 0, 0, 0, 0, 0, 0]);
                             _prefillArray[i] = (dynamic)new OrderDataUInt256(price, orderId, 1);
                         });
        }
        else { throw new Exception("Unsupported Type"); }
    }

    [IterationSetup(Target = "SortedDictionary_AddItem")]
    public void A_Iteration_Setup_SortedDictionary()
    {
        if (typeof(TData) == typeof(OrderDataBigInteger))
        {
            _collection =
                (dynamic)new SortedDictionary<OrderDataBigIntegerKey, OrderDataBigInteger>(((OrderDataBigInteger[])(dynamic)_prefillArray).ToDictionary(static pair =>
                                                                                                   new OrderDataBigIntegerKey(pair.Price, pair.OrderId),
                                                                                               static pair => pair));
        }
        else
        {
            _collection =
                (dynamic)new SortedDictionary<OrderDataUInt256Key, OrderDataUInt256>(((OrderDataUInt256[])(dynamic)_prefillArray).ToDictionary(static pair =>
                                                                                             new OrderDataUInt256Key(pair.Price, pair.OrderId),
                                                                                         static pair => pair));
        }
    }

    [IterationSetup(Target = "SortedImmutableDictionary_AddItem")]
    public void A_Iteration_Setup_SortedImmutableDictionary()
    {
        if (typeof(TData) == typeof(OrderDataBigInteger))
        {
            _collection =
                (dynamic)((OrderDataBigInteger[])(dynamic)_prefillArray).ToImmutableSortedDictionary(static p => new OrderDataBigIntegerKey(p.Price, p.OrderId), static p => p);
        }
        else
        {
            _collection = (dynamic)((OrderDataUInt256[])(dynamic)_prefillArray).ToImmutableSortedDictionary(static p => new OrderDataUInt256Key(p.Price, p.OrderId), static p => p);
        }
    }

    [IterationSetup(Target = "SortedList_AddItem")]
    public void A_Iteration_Setup_SortedList()
    {
        if (typeof(TData) == typeof(OrderDataBigInteger))
        {
            _collection =
                (dynamic)new SortedList<OrderDataBigIntegerKey, OrderDataBigInteger>(((OrderDataBigInteger[])(dynamic)_prefillArray).ToDictionary(static pair =>
                                                                                             new OrderDataBigIntegerKey(pair.Price, pair.OrderId),
                                                                                         static pair => pair));
        }
        else
        {
            _collection =
                (dynamic)new SortedList<OrderDataUInt256Key, OrderDataUInt256>(((OrderDataUInt256[])(dynamic)_prefillArray).ToDictionary(static pair =>
                                                                                       new OrderDataUInt256Key(pair.Price, pair.OrderId),
                                                                                   static pair => pair));
        }
    }

    [Benchmark(Baseline = true)]
    [BenchmarkCategory("AddItem")]
    public void SortedDictionary_AddItem()
    {
        BenchmarkHelper.FlatCollection(_orderDataKeys, _orderDataArray, ref _collection);
    }

    [Benchmark]
    [BenchmarkCategory("AddItem")]
    public void SortedImmutableDictionary_AddItem()
    {
        var immutable = (IImmutableDictionary<TKey, TData>)_collection;
        BenchmarkHelper.FlatImmutableCollection(_orderDataKeys, _orderDataArray, ref immutable);
    }

    [Benchmark]
    [BenchmarkCategory("AddItem")]
    public void SortedList_AddItem()
    {
        BenchmarkHelper.FlatCollection(_orderDataKeys, _orderDataArray, ref _collection);
    }

    // ReSharper disable NullableWarningSuppressionIsUsed
    private IDictionary<TKey, TData> _collection = default!;

    private TData[] _prefillArray = default!;

    public string[] GetDataTypeName { get; } = [typeof(TData) == typeof(OrderDataBigInteger) ? "BigInteger" : "UInt256"];

    [ParamsSource(nameof(GetDataTypeName), Priority = -1)]
    public string DataType { get; set; } = default!;
    // ReSharper restore NullableWarningSuppressionIsUsed

    // ReSharper disable NullableWarningSuppressionIsUsed
    private TData[] _orderDataArray = default!;

    private TKey[] _orderDataKeys = default!;
    // ReSharper restore NullableWarningSuppressionIsUsed
}

What I want to compare is the performance between the different variants. However I get the MinIterationTime warning. Since my time is far away from the needed minimum iteration time I think I need to rewrite the tests but I have absolutely no idea how to do that and achieve the same tests. What I do not want is to add the time to prepare the collection to the performance benchmark time.

[timcassell]

Add a const int OpCount = 100_000; Run the work in a loop for (int i = 0; i < OpCount; ++i). Attribute your benchmark like this [Benchmark(OperationsPerInvoke = OpCount)]. Adjust OpCount value until the warning goes away. If necessary, group your setup data into an array the size of OpCount, and loop to fill it in the [IterationSetup].

[adamsitnik]

A sample benchmark that does what @timcassell has described: https://github.com/dotnet/performance/blob/main/src/benchmarks/micro/libraries/System.Collections/Sort.cs

[swtrse]

thx that's what I was looking for. @timcassell I guess OperationsPerInvoke will calculate it down to time of one operation. Am I right?

[adamsitnik]

I guess OperationsPerInvoke will calculate it down to time of one operation. Am I right?

Yes, the results will be scaled.