🐞 Erratic connection reset errors when querying immudb on larger data sets

[ruescasd]

Preflight checklist

• [X] I could not find a solution in the existing issues, docs, nor discussions.
• [X] I have attached screenshots and/or the specific steps to reproduce this issue.
• [X] I have attached a minimal election.json configuration to reproduce this issue (provide one for each different use-case).

Current Behavior

---- test_protocol_immudb stdout ----
thread 'test_protocol_immudb' panicked at /workspaces/backend-services/packages/braid/src/test/protocol_test_immudb.rs:193:44:
called `Result::unwrap()` on an `Err` value: status: Unknown, message: "h2 protocol error: error reading a body from connection: connection reset", details: [], metadata: MetadataMap { headers: {"content-type": "application/grpc", "immudb-uuid": "cmjreq6c474c73ejpq50"} }

Caused by:
    0: error reading a body from connection: connection reset
    1: connection reset

Stack backtrace:
   0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from
   1: immu_board::board_client::BoardClient::get::{{closure}}
   2: braid::protocol2::board::immudb::ImmudbBoard::get_remote_messages::{{closure}}
   3: braid::protocol2::session::Session<C>::step::{{closure}}
   4: braid::test::protocol_test_immudb::run_protocol_test_immudb::{{closure}}::{{closure}}::{{closure}}
   5: tokio::runtime::task::core::Core<T,S>::poll
   6: tokio::runtime::task::harness::Harness<T,S>::poll
   7: tokio::runtime::scheduler::current_thread::CurrentThread::block_on
   8: core::ops::function::FnOnce::call_once
   9: core::ops::function::FnOnce::call_once
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/ops/function.rs:250:5
  10: test::__rust_begin_short_backtrace
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:627:18
  11: test::run_test_in_process::{{closure}}
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:650:60
  12: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/panic/unwind_safe.rs:272:9
  13: std::panicking::try::do_call
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:552:40
  14: std::panicking::try
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:516:19
  15: std::panic::catch_unwind
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panic.rs:142:14
  16: test::run_test_in_process
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:650:27
  17: test::run_test::{{closure}}
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:573:43
  18: test::run_test::{{closure}}
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:601:41
  19: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys_common/backtrace.rs:155:18
  20: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/thread/mod.rs:529:17
  21: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/panic/unwind_safe.rs:272:9
  22: std::panicking::try::do_call
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:552:40
  23: std::panicking::try
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:516:19
  24: std::panic::catch_unwind
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panic.rs:142:14
  25: std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/thread/mod.rs:528:30
  26: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/ops/function.rs:250:5
  27: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/alloc/src/boxed.rs:2015:9
  28: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/alloc/src/boxed.rs:2015:9
  29: std::sys::unix::thread::Thread::new::thread_start
             at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys/unix/thread.rs:108:17
  30: start_thread
  31: clone
stack backtrace:
   0:     0x55fc279c3ebc - std::backtrace_rs::backtrace::libunwind::trace::h7cae80a0c43fb6b8
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/../../backtrace/src/backtrace/libunwind.rs:104:5
   1:     0x55fc279c3ebc - std::backtrace_rs::backtrace::trace_unsynchronized::h80a007a4d210e9de
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x55fc279c3ebc - std::sys_common::backtrace::_print_fmt::h88dbc14703caf101
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys_common/backtrace.rs:68:5
   3:     0x55fc279c3ebc - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h2dec0f1afa852b3f
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x55fc279f1820 - core::fmt::rt::Argument::fmt::h940602fceb08c8ea
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/fmt/rt.rs:142:9
   5:     0x55fc279f1820 - core::fmt::write::h02180458b8be9806
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/fmt/mod.rs:1120:17
   6:     0x55fc279c101f - std::io::Write::write_fmt::hd43b9bbcfacc823d
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/io/mod.rs:1810:15
   7:     0x55fc279c3ca4 - std::sys_common::backtrace::_print::hec3c0c0a24ac4e1f
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys_common/backtrace.rs:47:5
   8:     0x55fc279c3ca4 - std::sys_common::backtrace::print::h9eaca07bac29371a
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys_common/backtrace.rs:34:9
   9:     0x55fc279c59d7 - std::panicking::default_hook::{{closure}}::h10146f18d3ed0603
  10:     0x55fc279c56c3 - std::panicking::default_hook::he2386c524bafaf7b
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:289:9
  11:     0x55fc2794a1f7 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::hf98a31a9e05ba565
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/alloc/src/boxed.rs:2029:9
  12:     0x55fc2794a1f7 - test::test_main::{{closure}}::hebc2048dff0493ff
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:138:21
  13:     0x55fc279c5fe8 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::hfe9492348c9f535d
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/alloc/src/boxed.rs:2029:9
  14:     0x55fc279c5fe8 - std::panicking::rust_panic_with_hook::h4b9fae887fc3a765
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:783:13
  15:     0x55fc279c5d3e - std::panicking::begin_panic_handler::{{closure}}::h2ad3dbd4b247dc26
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:657:13
  16:     0x55fc279c4386 - std::sys_common::backtrace::__rust_end_short_backtrace::hc731f82821e80a73
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys_common/backtrace.rs:171:18
  17:     0x55fc279c5aa2 - rust_begin_unwind
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:645:5
  18:     0x55fc2745f3b5 - core::panicking::panic_fmt::hb82f075491433500
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/panicking.rs:72:14
  19:     0x55fc2745f913 - core::result::unwrap_failed::he2c24d914fe3d28d
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/result.rs:1649:5
  20:     0x55fc274aae7e - braid::test::protocol_test_immudb::run_protocol_test_immudb::{{closure}}::h72e4d2bf4c9a2d42
  21:     0x55fc274d1903 - test_protocol::test_protocol_immudb::{{closure}}::h1a80dfb2411bea3f
  22:     0x55fc27698ad3 - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::h4bae70a0e28da47e
  23:     0x55fc2749d8c0 - core::ops::function::FnOnce::call_once::h4cf01ea8774a5926
  24:     0x55fc2794f84f - core::ops::function::FnOnce::call_once::hf7b10bc57885dbec
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/ops/function.rs:250:5
  25:     0x55fc2794f84f - test::__rust_begin_short_backtrace::h8337a841c2f856e0
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:627:18
  26:     0x55fc2794e865 - test::run_test_in_process::{{closure}}::ha139ce5e0d584344
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:650:60
  27:     0x55fc2794e865 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h2d9dd4cae340eded
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/panic/unwind_safe.rs:272:9
  28:     0x55fc2794e865 - std::panicking::try::do_call::had9779d51213a05d
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:552:40
  29:     0x55fc2794e865 - std::panicking::try::h0d4c92515dcd8622
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:516:19
  30:     0x55fc2794e865 - std::panic::catch_unwind::ha95538e45bf70b4d
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panic.rs:142:14
  31:     0x55fc2794e865 - test::run_test_in_process::h52a521747f7f5788
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:650:27
  32:     0x55fc2794e865 - test::run_test::{{closure}}::he4f3dcc5a5f6a022
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:573:43
  33:     0x55fc27916aa6 - test::run_test::{{closure}}::hdbfe2bba58817ef1
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/test/src/lib.rs:601:41
  34:     0x55fc27916aa6 - std::sys_common::backtrace::__rust_begin_short_backtrace::ha3ed2fc2102edcfc
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys_common/backtrace.rs:155:18
  35:     0x55fc2791bd27 - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hec17ce5201bdf705
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/thread/mod.rs:529:17
  36:     0x55fc2791bd27 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h1c3e4ba05d545174
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/panic/unwind_safe.rs:272:9
  37:     0x55fc2791bd27 - std::panicking::try::do_call::h43dd97d3335b9315
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:552:40
  38:     0x55fc2791bd27 - std::panicking::try::hd9213605004664fb
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panicking.rs:516:19
  39:     0x55fc2791bd27 - std::panic::catch_unwind::h11d490030d2bae47
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/panic.rs:142:14
  40:     0x55fc2791bd27 - std::thread::Builder::spawn_unchecked_::{{closure}}::hf90ff8e662fc863a
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/thread/mod.rs:528:30
  41:     0x55fc2791bd27 - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha559285696cffb11
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/core/src/ops/function.rs:250:5
  42:     0x55fc279cabd5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h6a315651827fad55
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/alloc/src/boxed.rs:2015:9
  43:     0x55fc279cabd5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h81c738cb43edf584
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/alloc/src/boxed.rs:2015:9
  44:     0x55fc279cabd5 - std::sys::unix::thread::Thread::new::thread_start::hd62307e699becb7b
                               at /rustc/eeff92ad32c2627876112ccfe812e19d38494087/library/std/src/sys/unix/thread.rs:108:17
  45:     0x7fae45064084 - start_thread
  46:     0x7fae450e6494 - clone
  47:                0x0 - <unknown>

### Expected Behavior

-

### Steps To Reproduce

Occurs erratically

Trustees = 3, threshold = 2, ciphertexts = 100k

/packages/braid$ cargo test immudb --release -- --include-ignored

### Environment

- **OS**: Mac OS X Ventura
- **Web browser**: Google Chrome 112.0.5615.137 (Official Build) (arm64)
- **Sequent Platform Version**: 10.0.0

### Tasks

```[tasklist]
# Tasks
- [ ] Issue has been triaged
- [ ] Successful replication of the defect
- [ ] Successful defect resolution merged

# Main PRs
- [ ] PR 1

# Stable PRs
- [ ] PR 1

[ruescasd]

You will run into size issues earlier unless you have this patch included:

@ruescasd ➜ /workspaces/backend-services/packages/immudb-rs (feat/meta-297/main ✗) $ git diff src/client.rs 
diff --git a/packages/immudb-rs/src/client.rs b/packages/immudb-rs/src/client.rs
index 20ed5260..dca0d2f8 100644
--- a/packages/immudb-rs/src/client.rs
+++ b/packages/immudb-rs/src/client.rs
@@ -30,7 +30,10 @@ pub type AsyncResponse<T> = Result<Response<T>>;
 impl Client {
     #[instrument(skip(password))]
     pub async fn new(server_url: &str, username: &str, password: &str) -> Result<Client> {
-        let client = ImmuServiceClient::connect(String::from(server_url)).await?;
+        let mut client = ImmuServiceClient::connect(String::from(server_url)).await?;
+        client = client.max_encoding_message_size(134217728);
+        client = client.max_decoding_message_size(134217728);
+