(Performance Issue) load test: Performance getting worse

[akiradeveloper]

• 3 nodes
• 1000 shards
• 10 writes and 1 read for each shard (so 11000 in a batch)

As time passes, the it starts to make errors in both r/w and the response time get worse to 30x from the beginning (when it's healthy).

$ ./target/release/mrbench -t 3600s -w10 -r1 -n3 -p1000 --enable-console
[tests/mrbench/src/main.rs:29:5] &opts = Opts {
    num_nodes: 3,
    num_shards: 1000,
    io_duration: 3600s,
    n_batch_writes: 10,
    n_batch_reads: 1,
    io_size: 1,
    enable_console: true,
}
cluster setup: 3.031983491s
io done. 1.919450397s. error=(w=0,r=0)
io done. 2.06802786s. error=(w=0,r=0)
io done. 1.923848609s. error=(w=0,r=0)
io done. 1.93027313s. error=(w=0,r=0)
io done. 1.839149705s. error=(w=0,r=0)
io done. 1.952110347s. error=(w=0,r=0)
io done. 1.903628843s. error=(w=0,r=0)
io done. 2.044248374s. error=(w=0,r=0)
io done. 1.994255656s. error=(w=0,r=0)
io done. 2.140820182s. error=(w=0,r=0)
io done. 1.922878616s. error=(w=0,r=0)
io done. 1.935054767s. error=(w=0,r=0)
io done. 2.016103504s. error=(w=0,r=0)
io done. 2.203109816s. error=(w=0,r=0)
io done. 2.068800627s. error=(w=0,r=0)
io done. 2.266075674s. error=(w=0,r=0)
io done. 2.134740102s. error=(w=0,r=0)
io done. 2.095684566s. error=(w=0,r=0)
io done. 2.208457288s. error=(w=0,r=0)
io done. 2.19623611s. error=(w=0,r=0)
io done. 2.246342348s. error=(w=0,r=0)
io done. 2.544315882s. error=(w=0,r=0)
io done. 2.1703016s. error=(w=0,r=0)
io done. 3.307228898s. error=(w=0,r=0)
io done. 3.673376254s. error=(w=0,r=0)
io done. 3.717346753s. error=(w=0,r=0)
io done. 2.985731409s. error=(w=0,r=0)
io done. 2.835135829s. error=(w=0,r=0)
io done. 3.156205015s. error=(w=0,r=0)
io done. 3.409799659s. error=(w=0,r=0)
io done. 3.277569387s. error=(w=0,r=0)
io done. 3.103952441s. error=(w=0,r=0)
io done. 2.700494958s. error=(w=0,r=0)
io done. 2.994851882s. error=(w=0,r=0)
io done. 3.315508683s. error=(w=0,r=0)
^A1io done. 3.375578967s. error=(w=0,r=0)
io done. 3.481768054s. error=(w=0,r=0)
io done. 3.410124916s. error=(w=0,r=0)
io done. 4.467868731s. error=(w=0,r=0)
io done. 3.639805263s. error=(w=0,r=0)
io done. 4.532211383s. error=(w=0,r=0)
io done. 2.914546844s. error=(w=0,r=0)
io done. 3.213640802s. error=(w=0,r=0)
io done. 3.091249797s. error=(w=0,r=0)
io done. 3.01570484s. error=(w=0,r=0)
io done. 5.033419878s. error=(w=0,r=0)
io done. 3.758152786s. error=(w=0,r=0)
io done. 3.527190642s. error=(w=0,r=0)
io done. 3.489511884s. error=(w=0,r=0)
io done. 3.835976032s. error=(w=0,r=0)
io done. 4.058974578s. error=(w=0,r=0)
io done. 4.154097732s. error=(w=0,r=0)
io done. 3.888851964s. error=(w=0,r=0)
io done. 4.1970381s. error=(w=0,r=0)
io done. 4.089231831s. error=(w=0,r=0)
io done. 4.995213574s. error=(w=0,r=0)
io done. 4.905852842s. error=(w=0,r=0)
io done. 4.957510291s. error=(w=0,r=0)
io done. 14.469088769s. error=(w=0,r=0)
io done. 15.127637412s. error=(w=0,r=0)
io done. 15.891995354s. error=(w=0,r=10)
io done. 69.193814788s. error=(w=4,r=0)
io done. 59.912824511s. error=(w=10,r=99)
io done. 70.195929398s. error=(w=150,r=104)
io done. 66.226476444s. error=(w=949,r=96)
io done. 67.804388281s. error=(w=1376,r=164)
io done. 66.440659284s. error=(w=1634,r=227)
io done. 69.458558143s. error=(w=1703,r=174)
io done. 67.332851086s. error=(w=1754,r=201)

[akiradeveloper]

I reduced the node to 1 and it reproduced. This means the issue is irrelevant to internode communications.

$ ./target/release/mrbench -t 3600s -w10 -r1 -n1 -p1000 --enable-console
[tests/mrbench/src/main.rs:29:5] &opts = Opts {
    num_nodes: 1,
    num_shards: 1000,
    io_duration: 3600s,
    n_batch_writes: 10,
    n_batch_reads: 1,
    io_size: 1,
    enable_console: true,
}
cluster setup: 3.257591273s
io done. 1.466790835s. error=(w=0,r=0)
io done. 1.441070357s. error=(w=0,r=0)
io done. 1.430453373s. error=(w=0,r=0)
io done. 1.495604251s. error=(w=0,r=0)
io done. 1.434796882s. error=(w=0,r=0)
io done. 1.432533735s. error=(w=0,r=0)
io done. 1.496360441s. error=(w=0,r=0)
io done. 1.384379667s. error=(w=0,r=0)
io done. 1.320207865s. error=(w=0,r=0)
io done. 1.385479462s. error=(w=0,r=0)
io done. 1.399168221s. error=(w=0,r=0)
io done. 1.35044534s. error=(w=0,r=0)
io done. 1.41739875s. error=(w=0,r=0)
io done. 1.333401596s. error=(w=0,r=0)
io done. 1.388375699s. error=(w=0,r=0)
io done. 1.703397329s. error=(w=0,r=0)
io done. 1.371841518s. error=(w=0,r=0)
io done. 1.39852567s. error=(w=0,r=0)
io done. 1.51162286s. error=(w=0,r=0)
io done. 1.517055209s. error=(w=0,r=0)
io done. 1.362147712s. error=(w=0,r=0)
io done. 1.578787153s. error=(w=0,r=0)
io done. 1.393231207s. error=(w=0,r=0)
io done. 1.558532146s. error=(w=0,r=0)
io done. 1.588101972s. error=(w=0,r=0)
io done. 1.595594293s. error=(w=0,r=0)
io done. 1.659068575s. error=(w=0,r=0)
io done. 1.503527306s. error=(w=0,r=0)
io done. 1.661115136s. error=(w=0,r=0)
io done. 1.590054651s. error=(w=0,r=0)
io done. 1.731196076s. error=(w=0,r=0)
io done. 1.842900638s. error=(w=0,r=0)
io done. 1.935066467s. error=(w=0,r=0)
io done. 1.862631168s. error=(w=0,r=0)
io done. 3.22427851s. error=(w=0,r=0)
io done. 2.0368118s. error=(w=0,r=0)
io done. 5.252376589s. error=(w=0,r=0)
io done. 2.443930298s. error=(w=0,r=0)
io done. 5.338615078s. error=(w=0,r=0)
io done. 2.665014813s. error=(w=0,r=0)
io done. 3.234579479s. error=(w=0,r=0)
io done. 3.773626806s. error=(w=0,r=0)
io done. 5.362294854s. error=(w=0,r=0)
io done. 2.777225686s. error=(w=0,r=0)
io done. 3.28683076s. error=(w=0,r=0)
io done. 15.92402968s. error=(w=0,r=6)
io done. 14.659365426s. error=(w=0,r=7)
io done. 5.297028896s. error=(w=0,r=0)
io done. 17.960611219s. error=(w=0,r=8)
io done. 5.564973003s. error=(w=0,r=0)
io done. 25.428362767s. error=(w=0,r=0)
io done. 61.263987623s. error=(w=0,r=487)
io done. 71.4391491s. error=(w=0,r=11)

[akiradeveloper]

Setting r=0. It still reproduced.

$ ./target/release/mrbench -t 3600s -w10 -r0 -n1 -p1000 --enable-console
[tests/mrbench/src/main.rs:29:5] &opts = Opts {
    num_nodes: 1,
    num_shards: 1000,
    io_duration: 3600s,
    n_batch_writes: 10,
    n_batch_reads: 0,
    io_size: 1,
    enable_console: true,
}
cluster setup: 1.05698179s
io done. 1.229508571s. error=(w=0,r=0)
io done. 1.265874705s. error=(w=0,r=0)
io done. 1.272754235s. error=(w=0,r=0)
io done. 1.203349587s. error=(w=0,r=0)
io done. 1.145523463s. error=(w=0,r=0)
io done. 1.188872235s. error=(w=0,r=0)
io done. 1.158444616s. error=(w=0,r=0)
io done. 1.204047954s. error=(w=0,r=0)
io done. 1.219685702s. error=(w=0,r=0)
io done. 1.212552374s. error=(w=0,r=0)
io done. 1.229036587s. error=(w=0,r=0)
io done. 1.166781941s. error=(w=0,r=0)
io done. 1.218336222s. error=(w=0,r=0)
io done. 1.148864073s. error=(w=0,r=0)
io done. 1.17113491s. error=(w=0,r=0)
io done. 1.185598572s. error=(w=0,r=0)
io done. 1.305864264s. error=(w=0,r=0)
io done. 1.279567578s. error=(w=0,r=0)
io done. 1.282712117s. error=(w=0,r=0)
io done. 1.254044543s. error=(w=0,r=0)
io done. 1.321884406s. error=(w=0,r=0)
io done. 1.721630501s. error=(w=0,r=0)
io done. 1.380776367s. error=(w=0,r=0)
io done. 1.376670968s. error=(w=0,r=0)
io done. 1.363863336s. error=(w=0,r=0)
io done. 1.435933692s. error=(w=0,r=0)
io done. 1.38645581s. error=(w=0,r=0)
io done. 1.506052745s. error=(w=0,r=0)
io done. 1.471685443s. error=(w=0,r=0)
io done. 1.54251729s. error=(w=0,r=0)
io done. 1.565267365s. error=(w=0,r=0)
io done. 1.617016301s. error=(w=0,r=0)
io done. 1.571293344s. error=(w=0,r=0)
io done. 1.533600994s. error=(w=0,r=0)
io done. 1.801617224s. error=(w=0,r=0)
io done. 1.735276185s. error=(w=0,r=0)
io done. 1.704838124s. error=(w=0,r=0)
io done. 1.785427396s. error=(w=0,r=0)
io done. 1.901012247s. error=(w=0,r=0)
io done. 1.772177958s. error=(w=0,r=0)
io done. 1.843704462s. error=(w=0,r=0)
io done. 1.84053512s. error=(w=0,r=0)
io done. 2.047842045s. error=(w=0,r=0)
io done. 2.103843519s. error=(w=0,r=0)
io done. 2.035020182s. error=(w=0,r=0)
io done. 2.588855976s. error=(w=0,r=0)
io done. 14.323160246s. error=(w=0,r=0)
io done. 3.948352686s. error=(w=0,r=0)
io done. 6.12443762s. error=(w=0,r=0)
io done. 13.688982096s. error=(w=0,r=0)
io done. 15.890372229s. error=(w=0,r=0)
io done. 57.309151176s. error=(w=0,r=0)
io done. 96.211637125s. error=(w=43,r=0)
io done. 96.373114007s. error=(w=6630,r=0)

[akiradeveloper]

Even with w=1, it reproduced (but no error count)

$ ./target/release/mrbench -t 3600s -w1 -r0 -n1 -p1000 --enable-console
[tests/mrbench/src/main.rs:29:5] &opts = Opts {
    num_nodes: 1,
    num_shards: 1000,
    io_duration: 3600s,
    n_batch_writes: 1,
    n_batch_reads: 0,
    io_size: 1,
    enable_console: true,
}
cluster setup: 1.713201385s
io done. 111.549842ms. error=(w=0,r=0)
io done. 127.864934ms. error=(w=0,r=0)
io done. 115.026987ms. error=(w=0,r=0)
io done. 117.278954ms. error=(w=0,r=0)
io done. 107.058074ms. error=(w=0,r=0)
io done. 124.904586ms. error=(w=0,r=0)
io done. 122.626973ms. error=(w=0,r=0)
io done. 109.295294ms. error=(w=0,r=0)
io done. 129.43394ms. error=(w=0,r=0)
io done. 116.525324ms. error=(w=0,r=0)
io done. 106.398122ms. error=(w=0,r=0)
io done. 135.183335ms. error=(w=0,r=0)
io done. 100.440657ms. error=(w=0,r=0)
io done. 120.303746ms. error=(w=0,r=0)
io done. 131.929757ms. error=(w=0,r=0)
io done. 104.443174ms. error=(w=0,r=0)
io done. 110.072977ms. error=(w=0,r=0)
io done. 114.996418ms. error=(w=0,r=0)
io done. 164.370505ms. error=(w=0,r=0)
io done. 117.603312ms. error=(w=0,r=0)
io done. 119.735743ms. error=(w=0,r=0)
io done. 124.533446ms. error=(w=0,r=0)
io done. 122.766873ms. error=(w=0,r=0)
io done. 106.221189ms. error=(w=0,r=0)
io done. 128.417295ms. error=(w=0,r=0)
io done. 116.487849ms. error=(w=0,r=0)
io done. 116.495408ms. error=(w=0,r=0)
io done. 130.094672ms. error=(w=0,r=0)
io done. 125.997953ms. error=(w=0,r=0)
io done. 126.638304ms. error=(w=0,r=0)
io done. 127.360647ms. error=(w=0,r=0)
io done. 112.300003ms. error=(w=0,r=0)
io done. 125.540393ms. error=(w=0,r=0)
io done. 119.870944ms. error=(w=0,r=0)
io done. 123.735147ms. error=(w=0,r=0)
io done. 113.54292ms. error=(w=0,r=0)
io done. 124.223936ms. error=(w=0,r=0)
io done. 120.919102ms. error=(w=0,r=0)
io done. 118.038128ms. error=(w=0,r=0)
io done. 136.849336ms. error=(w=0,r=0)
io done. 109.037883ms. error=(w=0,r=0)
io done. 125.780156ms. error=(w=0,r=0)
io done. 128.13259ms. error=(w=0,r=0)
io done. 110.128105ms. error=(w=0,r=0)
io done. 116.127075ms. error=(w=0,r=0)
io done. 104.963271ms. error=(w=0,r=0)
io done. 117.243671ms. error=(w=0,r=0)
io done. 111.291653ms. error=(w=0,r=0)
io done. 112.649558ms. error=(w=0,r=0)
io done. 120.306927ms. error=(w=0,r=0)
io done. 115.546266ms. error=(w=0,r=0)
io done. 117.981612ms. error=(w=0,r=0)
io done. 118.796257ms. error=(w=0,r=0)
io done. 115.838748ms. error=(w=0,r=0)
io done. 113.15582ms. error=(w=0,r=0)
io done. 120.065418ms. error=(w=0,r=0)
io done. 122.532004ms. error=(w=0,r=0)
io done. 114.923636ms. error=(w=0,r=0)
io done. 202.520045ms. error=(w=0,r=0)
io done. 127.497371ms. error=(w=0,r=0)
io done. 107.405913ms. error=(w=0,r=0)
io done. 108.480578ms. error=(w=0,r=0)
io done. 113.559129ms. error=(w=0,r=0)
io done. 117.428103ms. error=(w=0,r=0)
io done. 118.815282ms. error=(w=0,r=0)
io done. 120.297194ms. error=(w=0,r=0)
io done. 120.089426ms. error=(w=0,r=0)
io done. 116.140757ms. error=(w=0,r=0)
io done. 121.663145ms. error=(w=0,r=0)
io done. 107.133285ms. error=(w=0,r=0)
io done. 97.536728ms. error=(w=0,r=0)
io done. 128.483447ms. error=(w=0,r=0)
io done. 120.30504ms. error=(w=0,r=0)
io done. 124.360047ms. error=(w=0,r=0)
io done. 124.818519ms. error=(w=0,r=0)
io done. 113.165003ms. error=(w=0,r=0)
io done. 112.62201ms. error=(w=0,r=0)
io done. 110.083888ms. error=(w=0,r=0)
io done. 121.368465ms. error=(w=0,r=0)
io done. 115.694387ms. error=(w=0,r=0)
io done. 112.668465ms. error=(w=0,r=0)
io done. 100.125282ms. error=(w=0,r=0)
io done. 118.983074ms. error=(w=0,r=0)
io done. 120.073732ms. error=(w=0,r=0)
io done. 120.165797ms. error=(w=0,r=0)
io done. 117.547288ms. error=(w=0,r=0)
io done. 130.416097ms. error=(w=0,r=0)
io done. 101.69509ms. error=(w=0,r=0)
io done. 126.324537ms. error=(w=0,r=0)
io done. 109.201252ms. error=(w=0,r=0)
io done. 117.362247ms. error=(w=0,r=0)
io done. 116.68455ms. error=(w=0,r=0)
io done. 143.879888ms. error=(w=0,r=0)
io done. 113.642654ms. error=(w=0,r=0)
io done. 108.24969ms. error=(w=0,r=0)
io done. 122.911978ms. error=(w=0,r=0)
io done. 121.468042ms. error=(w=0,r=0)
io done. 119.794574ms. error=(w=0,r=0)
io done. 146.925947ms. error=(w=0,r=0)
io done. 117.500881ms. error=(w=0,r=0)
io done. 136.4473ms. error=(w=0,r=0)
io done. 124.85214ms. error=(w=0,r=0)
io done. 113.217444ms. error=(w=0,r=0)
io done. 119.665312ms. error=(w=0,r=0)
io done. 131.33928ms. error=(w=0,r=0)
io done. 121.916577ms. error=(w=0,r=0)
io done. 134.6798ms. error=(w=0,r=0)
io done. 135.4667ms. error=(w=0,r=0)
io done. 149.614551ms. error=(w=0,r=0)
io done. 117.612497ms. error=(w=0,r=0)
io done. 125.528121ms. error=(w=0,r=0)
io done. 112.610851ms. error=(w=0,r=0)
io done. 139.645962ms. error=(w=0,r=0)
io done. 118.888722ms. error=(w=0,r=0)
io done. 138.975197ms. error=(w=0,r=0)
io done. 125.459878ms. error=(w=0,r=0)
io done. 135.439554ms. error=(w=0,r=0)
io done. 104.196619ms. error=(w=0,r=0)
io done. 107.287079ms. error=(w=0,r=0)
io done. 104.803027ms. error=(w=0,r=0)
io done. 124.58962ms. error=(w=0,r=0)
io done. 113.562509ms. error=(w=0,r=0)
io done. 132.313244ms. error=(w=0,r=0)
io done. 114.977158ms. error=(w=0,r=0)
io done. 132.820539ms. error=(w=0,r=0)
io done. 129.142553ms. error=(w=0,r=0)
io done. 115.87742ms. error=(w=0,r=0)
io done. 125.222002ms. error=(w=0,r=0)
io done. 161.91438ms. error=(w=0,r=0)
io done. 129.073159ms. error=(w=0,r=0)
io done. 111.870261ms. error=(w=0,r=0)
io done. 118.3018ms. error=(w=0,r=0)
io done. 142.719131ms. error=(w=0,r=0)
io done. 112.296525ms. error=(w=0,r=0)
io done. 105.668428ms. error=(w=0,r=0)
io done. 117.603337ms. error=(w=0,r=0)
io done. 112.496022ms. error=(w=0,r=0)
io done. 120.395825ms. error=(w=0,r=0)
io done. 347.556696ms. error=(w=0,r=0)
io done. 136.971306ms. error=(w=0,r=0)
io done. 118.32113ms. error=(w=0,r=0)
io done. 120.415749ms. error=(w=0,r=0)
io done. 132.218431ms. error=(w=0,r=0)
io done. 134.483174ms. error=(w=0,r=0)
io done. 123.983039ms. error=(w=0,r=0)
io done. 133.263939ms. error=(w=0,r=0)
io done. 126.473804ms. error=(w=0,r=0)
io done. 121.41066ms. error=(w=0,r=0)
io done. 107.289764ms. error=(w=0,r=0)
io done. 124.021285ms. error=(w=0,r=0)
io done. 129.755097ms. error=(w=0,r=0)
io done. 111.978935ms. error=(w=0,r=0)
io done. 127.204913ms. error=(w=0,r=0)
io done. 128.769791ms. error=(w=0,r=0)
io done. 147.298121ms. error=(w=0,r=0)
io done. 116.772088ms. error=(w=0,r=0)
io done. 111.644682ms. error=(w=0,r=0)
io done. 110.465249ms. error=(w=0,r=0)
io done. 133.303029ms. error=(w=0,r=0)
io done. 138.424886ms. error=(w=0,r=0)
io done. 122.277633ms. error=(w=0,r=0)
io done. 124.850131ms. error=(w=0,r=0)
io done. 144.869471ms. error=(w=0,r=0)
io done. 127.310553ms. error=(w=0,r=0)
io done. 117.313657ms. error=(w=0,r=0)
io done. 138.279969ms. error=(w=0,r=0)
io done. 132.661375ms. error=(w=0,r=0)
io done. 115.726966ms. error=(w=0,r=0)
io done. 132.507095ms. error=(w=0,r=0)
io done. 153.401568ms. error=(w=0,r=0)
io done. 139.189803ms. error=(w=0,r=0)
io done. 122.58107ms. error=(w=0,r=0)
io done. 116.942936ms. error=(w=0,r=0)
io done. 130.199731ms. error=(w=0,r=0)
io done. 130.169497ms. error=(w=0,r=0)
io done. 152.15074ms. error=(w=0,r=0)
io done. 148.027079ms. error=(w=0,r=0)
io done. 134.782462ms. error=(w=0,r=0)
io done. 114.210812ms. error=(w=0,r=0)
io done. 117.938349ms. error=(w=0,r=0)
io done. 114.444484ms. error=(w=0,r=0)
io done. 131.713612ms. error=(w=0,r=0)
io done. 132.013782ms. error=(w=0,r=0)
io done. 129.155729ms. error=(w=0,r=0)
io done. 128.661327ms. error=(w=0,r=0)
io done. 148.582829ms. error=(w=0,r=0)
io done. 121.061433ms. error=(w=0,r=0)
io done. 110.04553ms. error=(w=0,r=0)
io done. 120.981687ms. error=(w=0,r=0)
io done. 122.891261ms. error=(w=0,r=0)
io done. 143.588125ms. error=(w=0,r=0)
io done. 162.444983ms. error=(w=0,r=0)
io done. 168.601406ms. error=(w=0,r=0)
io done. 119.26648ms. error=(w=0,r=0)
io done. 108.592142ms. error=(w=0,r=0)
io done. 122.703137ms. error=(w=0,r=0)
io done. 124.369033ms. error=(w=0,r=0)
io done. 143.443233ms. error=(w=0,r=0)
io done. 159.01856ms. error=(w=0,r=0)
io done. 146.749219ms. error=(w=0,r=0)
io done. 144.21958ms. error=(w=0,r=0)
io done. 113.107187ms. error=(w=0,r=0)
io done. 106.36519ms. error=(w=0,r=0)
io done. 117.771642ms. error=(w=0,r=0)
io done. 140.116703ms. error=(w=0,r=0)
io done. 153.921275ms. error=(w=0,r=0)
io done. 134.266036ms. error=(w=0,r=0)
io done. 169.874859ms. error=(w=0,r=0)
io done. 123.358181ms. error=(w=0,r=0)
io done. 113.631989ms. error=(w=0,r=0)
io done. 117.836613ms. error=(w=0,r=0)
io done. 133.106153ms. error=(w=0,r=0)
io done. 124.639816ms. error=(w=0,r=0)
io done. 156.204308ms. error=(w=0,r=0)
io done. 244.583741ms. error=(w=0,r=0)
io done. 134.771863ms. error=(w=0,r=0)
io done. 110.50395ms. error=(w=0,r=0)
io done. 112.138403ms. error=(w=0,r=0)
io done. 150.455982ms. error=(w=0,r=0)
io done. 143.310875ms. error=(w=0,r=0)
io done. 165.329741ms. error=(w=0,r=0)
io done. 184.490649ms. error=(w=0,r=0)
io done. 116.521958ms. error=(w=0,r=0)
io done. 125.811227ms. error=(w=0,r=0)
io done. 115.733366ms. error=(w=0,r=0)
io done. 167.186289ms. error=(w=0,r=0)
io done. 149.559073ms. error=(w=0,r=0)
io done. 169.209465ms. error=(w=0,r=0)
io done. 186.430018ms. error=(w=0,r=0)
io done. 116.725702ms. error=(w=0,r=0)
io done. 109.395431ms. error=(w=0,r=0)
io done. 123.394643ms. error=(w=0,r=0)
io done. 130.572647ms. error=(w=0,r=0)
io done. 156.593417ms. error=(w=0,r=0)
io done. 193.804216ms. error=(w=0,r=0)
io done. 179.578916ms. error=(w=0,r=0)
io done. 122.723044ms. error=(w=0,r=0)
io done. 118.723845ms. error=(w=0,r=0)
io done. 155.670506ms. error=(w=0,r=0)
io done. 159.094428ms. error=(w=0,r=0)
io done. 234.039384ms. error=(w=0,r=0)
io done. 181.222825ms. error=(w=0,r=0)
io done. 117.8001ms. error=(w=0,r=0)
io done. 123.287282ms. error=(w=0,r=0)
io done. 141.206042ms. error=(w=0,r=0)
io done. 152.634154ms. error=(w=0,r=0)
io done. 157.878615ms. error=(w=0,r=0)
io done. 147.000434ms. error=(w=0,r=0)
io done. 156.069495ms. error=(w=0,r=0)
io done. 130.374996ms. error=(w=0,r=0)
io done. 117.059464ms. error=(w=0,r=0)
io done. 145.356602ms. error=(w=0,r=0)
io done. 176.089916ms. error=(w=0,r=0)
io done. 240.252765ms. error=(w=0,r=0)
io done. 193.982596ms. error=(w=0,r=0)
io done. 118.817815ms. error=(w=0,r=0)
io done. 124.100678ms. error=(w=0,r=0)
io done. 147.789675ms. error=(w=0,r=0)
io done. 235.629436ms. error=(w=0,r=0)
io done. 208.198392ms. error=(w=0,r=0)
io done. 144.947852ms. error=(w=0,r=0)
io done. 138.904803ms. error=(w=0,r=0)
io done. 111.572157ms. error=(w=0,r=0)
io done. 137.604682ms. error=(w=0,r=0)
io done. 175.793002ms. error=(w=0,r=0)
io done. 187.864054ms. error=(w=0,r=0)
io done. 211.286564ms. error=(w=0,r=0)
io done. 130.452601ms. error=(w=0,r=0)
io done. 121.981219ms. error=(w=0,r=0)
io done. 154.827978ms. error=(w=0,r=0)
io done. 277.071318ms. error=(w=0,r=0)
io done. 248.394786ms. error=(w=0,r=0)
io done. 158.893071ms. error=(w=0,r=0)
io done. 127.520335ms. error=(w=0,r=0)
io done. 139.554851ms. error=(w=0,r=0)
io done. 162.57115ms. error=(w=0,r=0)
io done. 163.562027ms. error=(w=0,r=0)
io done. 288.878597ms. error=(w=0,r=0)
io done. 150.340463ms. error=(w=0,r=0)
io done. 118.467925ms. error=(w=0,r=0)
io done. 139.391841ms. error=(w=0,r=0)
io done. 196.266222ms. error=(w=0,r=0)
io done. 219.611047ms. error=(w=0,r=0)
io done. 194.095551ms. error=(w=0,r=0)
io done. 119.767615ms. error=(w=0,r=0)
io done. 117.096476ms. error=(w=0,r=0)
io done. 122.63796ms. error=(w=0,r=0)
io done. 193.604094ms. error=(w=0,r=0)
io done. 273.35472ms. error=(w=0,r=0)
io done. 204.826732ms. error=(w=0,r=0)
io done. 106.535289ms. error=(w=0,r=0)
io done. 127.402318ms. error=(w=0,r=0)
io done. 137.190411ms. error=(w=0,r=0)
io done. 180.209538ms. error=(w=0,r=0)
io done. 290.529487ms. error=(w=0,r=0)
io done. 225.872088ms. error=(w=0,r=0)
io done. 127.909608ms. error=(w=0,r=0)
io done. 180.79224ms. error=(w=0,r=0)
io done. 1.18957387s. error=(w=0,r=0)
io done. 499.050107ms. error=(w=0,r=0)
io done. 119.781159ms. error=(w=0,r=0)
io done. 164.52688ms. error=(w=0,r=0)
io done. 217.392339ms. error=(w=0,r=0)
io done. 304.605398ms. error=(w=0,r=0)
io done. 174.084764ms. error=(w=0,r=0)
io done. 118.001417ms. error=(w=0,r=0)
io done. 125.779324ms. error=(w=0,r=0)
io done. 313.953487ms. error=(w=0,r=0)
io done. 367.122325ms. error=(w=0,r=0)
io done. 133.566519ms. error=(w=0,r=0)
io done. 114.154247ms. error=(w=0,r=0)
io done. 145.06739ms. error=(w=0,r=0)
io done. 268.349241ms. error=(w=0,r=0)
io done. 334.464141ms. error=(w=0,r=0)
io done. 133.396455ms. error=(w=0,r=0)
io done. 116.798807ms. error=(w=0,r=0)
io done. 152.652429ms. error=(w=0,r=0)
io done. 184.414532ms. error=(w=0,r=0)
io done. 423.349608ms. error=(w=0,r=0)
io done. 142.830304ms. error=(w=0,r=0)
io done. 126.290344ms. error=(w=0,r=0)
io done. 180.652486ms. error=(w=0,r=0)
io done. 255.94349ms. error=(w=0,r=0)
io done. 229.454174ms. error=(w=0,r=0)
io done. 132.895582ms. error=(w=0,r=0)
io done. 108.097918ms. error=(w=0,r=0)
io done. 131.108745ms. error=(w=0,r=0)
io done. 233.245212ms. error=(w=0,r=0)
io done. 483.580778ms. error=(w=0,r=0)
io done. 137.368858ms. error=(w=0,r=0)
io done. 144.876305ms. error=(w=0,r=0)
io done. 258.601642ms. error=(w=0,r=0)
io done. 419.194676ms. error=(w=0,r=0)
io done. 131.485727ms. error=(w=0,r=0)
io done. 120.371562ms. error=(w=0,r=0)
io done. 279.98092ms. error=(w=0,r=0)
io done. 474.988579ms. error=(w=0,r=0)
io done. 128.632513ms. error=(w=0,r=0)
io done. 119.209002ms. error=(w=0,r=0)
io done. 220.422957ms. error=(w=0,r=0)
io done. 531.61919ms. error=(w=0,r=0)
io done. 132.588808ms. error=(w=0,r=0)
io done. 123.012117ms. error=(w=0,r=0)
io done. 389.756805ms. error=(w=0,r=0)
io done. 410.571909ms. error=(w=0,r=0)
io done. 121.202427ms. error=(w=0,r=0)
io done. 303.632441ms. error=(w=0,r=0)
io done. 491.814878ms. error=(w=0,r=0)
io done. 133.590152ms. error=(w=0,r=0)
io done. 138.794537ms. error=(w=0,r=0)
io done. 260.974729ms. error=(w=0,r=0)
io done. 453.31907ms. error=(w=0,r=0)
io done. 132.571351ms. error=(w=0,r=0)
io done. 118.646254ms. error=(w=0,r=0)
io done. 356.005944ms. error=(w=0,r=0)
io done. 375.769276ms. error=(w=0,r=0)
io done. 146.152065ms. error=(w=0,r=0)
io done. 130.13546ms. error=(w=0,r=0)
io done. 306.268622ms. error=(w=0,r=0)
io done. 481.268859ms. error=(w=0,r=0)
io done. 128.660806ms. error=(w=0,r=0)
io done. 156.810786ms. error=(w=0,r=0)
io done. 388.970787ms. error=(w=0,r=0)
io done. 330.402226ms. error=(w=0,r=0)
io done. 120.786897ms. error=(w=0,r=0)
io done. 163.281546ms. error=(w=0,r=0)
io done. 231.964703ms. error=(w=0,r=0)
io done. 460.443817ms. error=(w=0,r=0)
io done. 133.020549ms. error=(w=0,r=0)
io done. 140.76359ms. error=(w=0,r=0)
io done. 193.15136ms. error=(w=0,r=0)
io done. 474.452288ms. error=(w=0,r=0)
io done. 157.696031ms. error=(w=0,r=0)
io done. 122.232158ms. error=(w=0,r=0)
io done. 205.885337ms. error=(w=0,r=0)
io done. 642.687853ms. error=(w=0,r=0)
io done. 163.349847ms. error=(w=0,r=0)
io done. 207.494067ms. error=(w=0,r=0)
io done. 522.626867ms. error=(w=0,r=0)
io done. 137.659192ms. error=(w=0,r=0)
io done. 155.808109ms. error=(w=0,r=0)
io done. 512.764017ms. error=(w=0,r=0)
io done. 265.443851ms. error=(w=0,r=0)
io done. 124.859751ms. error=(w=0,r=0)
io done. 203.756199ms. error=(w=0,r=0)
io done. 392.80122ms. error=(w=0,r=0)
io done. 224.508555ms. error=(w=0,r=0)
io done. 129.46813ms. error=(w=0,r=0)
io done. 233.963705ms. error=(w=0,r=0)
io done. 723.983635ms. error=(w=0,r=0)
io done. 167.27527ms. error=(w=0,r=0)
io done. 482.550303ms. error=(w=0,r=0)
io done. 384.669279ms. error=(w=0,r=0)
io done. 137.74501ms. error=(w=0,r=0)
io done. 735.274941ms. error=(w=0,r=0)
io done. 157.808151ms. error=(w=0,r=0)
io done. 268.100475ms. error=(w=0,r=0)
io done. 498.160702ms. error=(w=0,r=0)
io done. 176.100563ms. error=(w=0,r=0)
io done. 125.301294ms. error=(w=0,r=0)
io done. 443.327932ms. error=(w=0,r=0)
io done. 359.519933ms. error=(w=0,r=0)
io done. 117.359985ms. error=(w=0,r=0)
io done. 204.549879ms. error=(w=0,r=0)
io done. 695.365654ms. error=(w=0,r=0)
io done. 120.466234ms. error=(w=0,r=0)
io done. 194.244132ms. error=(w=0,r=0)
io done. 679.89526ms. error=(w=0,r=0)
io done. 154.842602ms. error=(w=0,r=0)
io done. 259.608187ms. error=(w=0,r=0)
io done. 653.100978ms. error=(w=0,r=0)
io done. 203.594065ms. error=(w=0,r=0)
io done. 655.940993ms. error=(w=0,r=0)
io done. 164.270578ms. error=(w=0,r=0)
io done. 200.903238ms. error=(w=0,r=0)
io done. 663.03338ms. error=(w=0,r=0)
io done. 157.938461ms. error=(w=0,r=0)
io done. 173.579002ms. error=(w=0,r=0)
io done. 297.363262ms. error=(w=0,r=0)
io done. 383.169175ms. error=(w=0,r=0)
io done. 140.567631ms. error=(w=0,r=0)
io done. 291.106106ms. error=(w=0,r=0)
io done. 658.367161ms. error=(w=0,r=0)
io done. 393.614661ms. error=(w=0,r=0)
io done. 594.134183ms. error=(w=0,r=0)
io done. 205.04242ms. error=(w=0,r=0)
io done. 677.211785ms. error=(w=0,r=0)
io done. 176.042235ms. error=(w=0,r=0)
io done. 223.081805ms. error=(w=0,r=0)
io done. 673.907152ms. error=(w=0,r=0)
io done. 152.199534ms. error=(w=0,r=0)
io done. 365.094676ms. error=(w=0,r=0)
io done. 467.703357ms. error=(w=0,r=0)
io done. 124.217796ms. error=(w=0,r=0)
io done. 209.129664ms. error=(w=0,r=0)
io done. 661.20387ms. error=(w=0,r=0)
io done. 125.98256ms. error=(w=0,r=0)
io done. 436.203173ms. error=(w=0,r=0)
io done. 470.665374ms. error=(w=0,r=0)
io done. 141.886113ms. error=(w=0,r=0)
io done. 262.181902ms. error=(w=0,r=0)
io done. 487.755251ms. error=(w=0,r=0)
io done. 130.022807ms. error=(w=0,r=0)
io done. 144.39665ms. error=(w=0,r=0)
io done. 614.442937ms. error=(w=0,r=0)
io done. 274.593588ms. error=(w=0,r=0)
io done. 199.381314ms. error=(w=0,r=0)
io done. 663.067631ms. error=(w=0,r=0)
io done. 164.1498ms. error=(w=0,r=0)
io done. 339.141139ms. error=(w=0,r=0)
io done. 652.583502ms. error=(w=0,r=0)
io done. 885.417729ms. error=(w=0,r=0)
io done. 203.84225ms. error=(w=0,r=0)
io done. 430.64628ms. error=(w=0,r=0)
io done. 398.862367ms. error=(w=0,r=0)
io done. 352.584713ms. error=(w=0,r=0)
io done. 671.571809ms. error=(w=0,r=0)
io done. 389.094574ms. error=(w=0,r=0)
io done. 635.459095ms. error=(w=0,r=0)
io done. 885.681978ms. error=(w=0,r=0)
io done. 1.014016296s. error=(w=0,r=0)
io done. 661.566798ms. error=(w=0,r=0)
io done. 454.100942ms. error=(w=0,r=0)
io done. 741.521121ms. error=(w=0,r=0)
io done. 430.292588ms. error=(w=0,r=0)
io done. 861.969182ms. error=(w=0,r=0)
io done. 979.723146ms. error=(w=0,r=0)
io done. 874.68404ms. error=(w=0,r=0)
io done. 244.083629ms. error=(w=0,r=0)
io done. 802.543004ms. error=(w=0,r=0)
io done. 288.635646ms. error=(w=0,r=0)
io done. 652.758505ms. error=(w=0,r=0)
io done. 128.031524ms. error=(w=0,r=0)
io done. 462.04291ms. error=(w=0,r=0)
io done. 509.802263ms. error=(w=0,r=0)
io done. 476.169948ms. error=(w=0,r=0)
io done. 814.416589ms. error=(w=0,r=0)
io done. 883.25177ms. error=(w=0,r=0)
io done. 846.906286ms. error=(w=0,r=0)
io done. 836.082487ms. error=(w=0,r=0)
io done. 335.13839ms. error=(w=0,r=0)
io done. 796.47839ms. error=(w=0,r=0)
io done. 654.054455ms. error=(w=0,r=0)
io done. 378.569342ms. error=(w=0,r=0)
io done. 1.010089315s. error=(w=0,r=0)
io done. 5.533649307s. error=(w=0,r=0)
io done. 5.332277584s. error=(w=0,r=0)
io done. 571.710529ms. error=(w=0,r=0)
io done. 863.469171ms. error=(w=0,r=0)
io done. 790.002664ms. error=(w=0,r=0)
io done. 1.773990697s. error=(w=0,r=0)
io done. 1.347238339s. error=(w=0,r=0)
io done. 1.333528668s. error=(w=0,r=0)
io done. 5.522327517s. error=(w=0,r=0)
io done. 6.381204158s. error=(w=0,r=0)
io done. 6.456759817s. error=(w=0,r=0)
io done. 10.851549784s. error=(w=0,r=0)
io done. 5.23342915s. error=(w=0,r=0)
io done. 1.672066292s. error=(w=0,r=0)
io done. 1.321328292s. error=(w=0,r=0)
io done. 7.873035296s. error=(w=0,r=0)
io done. 6.951577263s. error=(w=0,r=0)
io done. 11.007766588s. error=(w=0,r=0)
io done. 5.390397966s. error=(w=0,r=0)
io done. 5.687078031s. error=(w=0,r=0)
io done. 5.33258526s. error=(w=0,r=0)
io done. 5.462663159s. error=(w=0,r=0)
io done. 5.221862156s. error=(w=0,r=0)
io done. 292.715023ms. error=(w=0,r=0)
io done. 668.833733ms. error=(w=0,r=0)
io done. 357.767678ms. error=(w=0,r=0)
io done. 845.160085ms. error=(w=0,r=0)
io done. 1.715180029s. error=(w=0,r=0)
io done. 1.214084707s. error=(w=0,r=0)
io done. 1.407248484s. error=(w=0,r=0)
io done. 5.474206134s. error=(w=0,r=0)
io done. 6.901615347s. error=(w=0,r=0)
io done. 6.417537629s. error=(w=0,r=0)
io done. 12.209446881s. error=(w=0,r=0)
io done. 13.35096909s. error=(w=0,r=0)
io done. 6.196791974s. error=(w=0,r=0)
io done. 11.29029968s. error=(w=0,r=0)
io done. 11.766667811s. error=(w=0,r=0)
io done. 8.046434985s. error=(w=0,r=0)
io done. 23.956449562s. error=(w=0,r=0)
io done. 13.086396585s. error=(w=0,r=0)
io done. 13.586793146s. error=(w=0,r=0)
io done. 7.27339722s. error=(w=0,r=0)
io done. 7.232668935s. error=(w=0,r=0)
io done. 10.92228831s. error=(w=0,r=0)
io done. 7.825054011s. error=(w=0,r=0)
io done. 16.723878587s. error=(w=0,r=0)
io done. 110.332925ms. error=(w=0,r=0)
io done. 155.475764ms. error=(w=0,r=0)
io done. 625.845839ms. error=(w=0,r=0)
io done. 246.004622ms. error=(w=0,r=0)
io done. 206.645776ms. error=(w=0,r=0)
io done. 838.660493ms. error=(w=0,r=0)
io done. 926.295614ms. error=(w=0,r=0)
io done. 268.489806ms. error=(w=0,r=0)
io done. 790.368177ms. error=(w=0,r=0)
io done. 1.174023668s. error=(w=0,r=0)
io done. 12.524967821s. error=(w=0,r=0)
io done. 12.823036732s. error=(w=0,r=0)
io done. 9.569363558s. error=(w=0,r=0)
io done. 15.350889102s. error=(w=0,r=0)
io done. 9.517126281s. error=(w=0,r=0)
io done. 13.046112181s. error=(w=0,r=0)
io done. 12.59332495s. error=(w=0,r=0)
io done. 8.024089502s. error=(w=0,r=0)
io done. 5.612946964s. error=(w=0,r=0)
io done. 5.724327369s. error=(w=0,r=0)
io done. 7.575709223s. error=(w=0,r=0)
io done. 11.990508962s. error=(w=0,r=0)
io done. 21.86478883s. error=(w=0,r=0)

[akiradeveloper]

$ ./target/release/mrbench -t 3600s -w10 -r0 -n1 -p1000 --enable-console                                                                                                                                
[tests/mrbench/src/main.rs:29:5] &opts = Opts {
    num_nodes: 1,
    num_shards: 1000,
    io_duration: 3600s,
    n_batch_writes: 10,
    n_batch_reads: 0,
    io_size: 1,
    enable_console: true,
}

My guess was ceaseless inserts to redb (mem backed) makes it slow down because of excessive number of entries.

So, I changed mrbench to send the server make-snapshot request (which leads to compaction) after each IO batch.

        let mut futs = vec![];
        for shard_id in 0..opts.num_shards {
            let cli = cluster.user(0);
            let fut = async move {
                cli.make_snapshot(shard_id).await
            };
            futs.push(fut);
        }
        futures::future::try_join_all(futs).await?;

Fortunately, this resolved the issue.

I will write a benchmark program to measure the sustain performance of redb.