Server crashes when using glommio

[pionoor]

Hi,

I'm prototyping a service, decided to try out ntex. I stripped out irrelevant logic kept the logic that is triggering the crash. Im using vegeta to load test the server using tokio and glommio runtimes. With tokio runtime it does not crash, with glommio it does after a certain threshold which again, it does not happen with tokio..

#[web::get("/resize/{encodedUrl}/")]
async fn resize(path: web::types::Path<(String)>, state: State<AppState>) -> impl web::Responder {
    let img_url = path.into_inner();

    // decode base64
    let decoded = base64_url::decode(&img_url).unwrap();
    let url = std::str::from_utf8(&decoded).unwrap();

    // extract image extension and original image name
    let url_parts: Vec<&str> = url.split('/').collect();
    let file_name = url_parts.last().unwrap_or(&"resized.png");
    let file_parts: Vec<&str> = file_name.split('.').collect();
    let ext = format!(".{}", file_parts.last().unwrap_or(&"png"));
    let image_data = download_image(url).await.unwrap();

    //let resized_image = resize_libvip(image_data, &ext).await.unwrap();
    web::HttpResponse::Ok()
        .content_type(format!("image/{}", ext))
        .body("ok")
}

async fn download_image(url: &str) -> Result<Vec<u8>> {
    let mut client = Client::default();
    let mut response = client
        .get(url) // <- Create request builder
        .header("User-Agent", "ntex::web")
        .send() // <- Send http request
        .await
        .map_err(|_| anyhow!("failed to create http client"))?;

    let body = response.body().await?;
    let body_bytes: Bytes = body.into();
    let body_vec: Vec<u8> = body_bytes.to_vec();

    Ok(body_vec)
}

#[ntex::main]
async fn main() -> std::io::Result<()> {
    // let img_url = "https://display-events.adgreetz.com/sample.png";
    // load TLS keys
    // to create a self-signed temporary cert for testing:
    // if std::env::var("RUST_LOG").is_err() {
    //     std::env::set_var("RUST_LOG", "info");
    // }
    // env_logger::init();
    //
    // let mut builder = SslAcceptor::mozilla_intermediate(SslMethod::tls()).unwrap();
    // builder
    //     .set_private_key_file("key.pem", SslFiletype::PEM)
    //     .unwrap();
    // builder.set_certificate_chain_file("cert.pem").unwrap();

    web::HttpServer::new(|| {
        web::App::new()
            .state(AppState {
                vips: {
                    let vips = VipsApp::new("Libvips", true).unwrap();
                    vips.concurrency_set(2);
                    vips
                },
                http_client: Client::default(),
            })
            .service(resize)
    })
    // .bind_rustls(("0.0.0.0", 8080), config)?
    .bind("0.0.0.0:8080")?
    .run()
    .await
}

Logs:

     Running `target/release/image-resizer`
thread 'ntex-rt:worker:0' panicked at 'already mutably borrowed: BorrowError', /root/.cargo/registry/src/github.com-1ecc6299db9ec823/ntex-glommio-0.2.3/src/io.rs:182:34
stack backtrace:
   0: rust_begin_unwind
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
   1: core::panicking::panic_fmt
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
   2: core::result::unwrap_failed
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/result.rs:1750:5
   3: <ntex_glommio::io::WriteTask as core::future::future::Future>::poll::{{closure}}
   4: <ntex_glommio::io::WriteTask as core::future::future::Future>::poll
   5: glommio::task::raw::RawTask<F,R,S>::run
   6: glommio::executor::LocalExecutor::run_task_queues
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
thread 'ntex-rt:worker:0' panicked at 'cannot access a scoped thread local variable without calling `set` first', /root/.cargo/registry/src/github.com-1ecc6299db9ec823/scoped-tls-1.0.1/src/lib.rs:168:9
stack backtrace:
   0:     0x55965850c59d - std::backtrace_rs::backtrace::libunwind::trace::ha9053a9a07ca49cb
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0x55965850c59d - std::backtrace_rs::backtrace::trace_unsynchronized::h9c2852a457ad564e
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x55965850c59d - std::sys_common::backtrace::_print_fmt::h457936fbfaa0070f
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:65:5
   3:     0x55965850c59d - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h5779d7bf7f70cb0c
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x5596582b54ae - core::fmt::write::h5a4baaff1bcd3eb5
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/fmt/mod.rs:1232:17
   5:     0x5596584e69f2 - std::io::Write::write_fmt::h4bc1f301cb9e9cce
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/io/mod.rs:1684:15
   6:     0x55965850dd8f - std::sys_common::backtrace::_print::h5fcdc36060f177e8
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:47:5
   7:     0x55965850dd8f - std::sys_common::backtrace::print::h54ca9458b876c8bf
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:34:9
   8:     0x55965850d98d - std::panicking::default_hook::{{closure}}::hbe471161c7664ed6
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:271:22
   9:     0x55965850e9ec - std::panicking::default_hook::ha3500da57aa4ac4f
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:290:9
  10:     0x55965850e9ec - std::panicking::rust_panic_with_hook::h50c09d000dc561d2
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:692:13
  11:     0x5596583fe7eb - std::panicking::begin_panic::{{closure}}::h8818964184dda406
  12:     0x5596583fe7b6 - std::sys_common::backtrace::__rust_end_short_backtrace::h61520f00cd616f0a
  13:     0x55965825e68a - std::panicking::begin_panic::hedf9b16a80c7a0fd
  14:     0x5596583fc62a - ntex_h2::stream::StreamState::review_state::h195b64f5da82d6d1
  15:     0x559658335adb - core::ptr::drop_in_place<core::option::Option<ntex_h2::stream::Stream>>::h50c95bc3cf3a3ebe
  16:     0x5596583358ea - <alloc::rc::Rc<T> as core::ops::drop::Drop>::drop::hb69df5c024659675
  17:     0x55965831907e - core::ptr::drop_in_place<ntex::http::client::response::MessageBody>::h29b8924c3d962f34
  18:     0x559658319321 - core::ptr::drop_in_place<image_resizer::download_image::{{closure}}>::ha81e3cb38b76782f
  19:     0x5596583198be - core::ptr::drop_in_place<<image_resizer::resize as ntex::web::service::WebServiceFactory<ntex::web::error_default::DefaultError>>::register::resize::{{closure}}>::h8f09051059965d02
  20:     0x559658312841 - core::ptr::drop_in_place<<ntex::web::handler::HandlerWrapper<<image_resizer::resize as ntex::web::service::WebServiceFactory<ntex::web::error_default::DefaultError>>::register::resize,(ntex::web::types::path::Path<alloc::string::String>,ntex::web::types::state::State<image_resizer::AppState>),ntex::web::error_default::DefaultError> as ntex::web::handler::HandlerFn<ntex::web::error_default::DefaultError>>::call::{{closure}}>::h8eca44c331b6e715
  21:     0x559658322296 - core::ptr::drop_in_place<ntex_service::and_then::State<ntex::web::app::Filter<ntex::web::error_default::DefaultError>,ntex::web::resource::ResourceRouter<ntex::web::error_default::DefaultError>,ntex::web::request::WebRequest<ntex::web::error_default::DefaultError>>>::h507104c0ffe8a5de
  22:     0x559658306b5d - core::ptr::drop_in_place<ntex::http::service::HttpServiceHandlerResponse<ntex_io::filter::Base,ntex::web::app_service::AppFactoryService<ntex::web::app_service::AppService<ntex::web::app::Filter<ntex::web::error_default::DefaultError>,ntex::web::error_default::DefaultError>,ntex::web::error_default::DefaultError>,ntex::http::body::Body,ntex::http::h1::expect::ExpectHandler,ntex::http::h1::upgrade::UpgradeHandler<ntex_io::filter::Base>>>::h7653feffbcc9f71e
  23:     0x5596583067eb - core::ptr::drop_in_place<<ntex::server::service::StreamService<ntex::http::service::HttpServiceHandler<ntex_io::filter::Base,ntex::web::app_service::AppFactoryService<ntex::web::app_service::AppService<ntex::web::app::Filter<ntex::web::error_default::DefaultError>,ntex::web::error_default::DefaultError>,ntex::web::error_default::DefaultError>,ntex::http::body::Body,ntex::http::h1::expect::ExpectHandler,ntex::http::h1::upgrade::UpgradeHandler<ntex_io::filter::Base>>> as ntex_service::Service<(core::option::Option<ntex::server::counter::CounterGuard>,ntex::server::service::ServerMessage)>>::call::{{closure}}>::hbc7bea149021ffbc.5976
  24:     0x55965830b11a - glommio::task::raw::RawTask<F,R,S>::drop_future::h2d3ea3139ff5e08b
  25:     0x5596583f53c7 - <alloc::collections::vec_deque::VecDeque<T,A> as core::ops::drop::Drop>::drop::h049cbd5352330135
  26:     0x559658423b0d - <alloc::rc::Rc<T> as core::ops::drop::Drop>::drop::h0486b24b46846866
  27:     0x5596584256f8 - core::ptr::drop_in_place<glommio::executor::LocalExecutor>::hdcc40b35371d8ee6
  28:     0x55965842e10b - std::sys_common::backtrace::__rust_begin_short_backtrace::haf9485c9ab8f1d36
  29:     0x559658426506 - core::ops::function::FnOnce::call_once{{vtable.shim}}::hacabd6f75f070c99
  30:     0x55965850f775 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h39990b24eedef2ab
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
  31:     0x55965850f775 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h01a027258444143b
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
  32:     0x55965850f775 - std::sys::unix::thread::Thread::new::thread_start::ha4f1cdd9c25884ba
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys/unix/thread.rs:108:17
  33:     0x7f1c5c88f18a - <unknown>
  34:     0x7f1c5c91dbd0 - <unknown>
  35:                0x0 - <unknown>
thread panicked while panicking. aborting.
Aborted (core dumped)

[fafhrd91]

do you have backtrace or error?

[leon3s]

Could it be because of the HttpClient that carry an Rc pointer and cannot be share between threads ? I wasn't able to pass it as a state once builded, seems you are doing something like this

[pionoor]

do you have backtrace or error?

It is on the there!

[pionoor]

Could it be because of the HttpClient that carry an Rc pointer and cannot be share between threads ? I wasn't able to pass it as a state once builded, seems you are doing something like this

Well it should be the same for tokio multi threaded? In tokio it does not crash, just in glommio.

So what do you recommend, just arc the client and add it to the app state?

[pionoor]

Also forgot to mention, the server crash during load testing. I was load testing both runtimes to see if there is any difference. Tokio does not crash. Glommio crash when i do more than 50 request/second. So i don't think it is Rc issue.

[fafhrd91]

do you have backtrace for crash?

[fafhrd91]

oh, right

[fafhrd91]

could you try this "glommio-borrow-fix" branch

[pionoor]

could you try this "glommio-borrow-fix" branch

Thank you! I will try tomorrow and let you know.

[pionoor]

It seems like the fix helped a bit, the server now is accepting more requests but still eventually crash when i do perf load 100 request/second. I have made some changes to the code, I moved the http client that makes the external request to the app state. main.rs:

use std::io::{BufReader, BufWriter, Cursor};
use std::num::NonZeroU32;
use ntex::web;
use image::codecs::png::PngEncoder;
use image::io::Reader as ImageReader;
use image::{ColorType, ImageEncoder};
use libvips::{ops, VipsImage, VipsApp};
use std::time::{Instant, Duration};
// use ntex_files::Files;
use rustls::{Certificate, PrivateKey, ServerConfig};
use rustls_pemfile::{certs, rsa_private_keys};
use fast_image_resize as fr;
use ntex::web::types::State;
use ntex::http::client::{Client, ClientBuilder};

async fn resize_libvip(http_client: &Client, url: &str, image_extension: &str) -> Result<Vec<u8>, Box< dyn std::error::Error>>{

    let image_data = download_image(http_client, url).await?;
    let image = VipsImage::new_from_buffer(&image_data, "").unwrap();
    let start = Instant::now();
    let resized = ops::thumbnail_image(&image, 800).unwrap();
    let end = Instant::now();
    let elapsed = end.duration_since(start);
    // Convert the difference to milliseconds.
    let elapsed_ms = elapsed.as_micros();
    println!("Vips execution time: {} ms", elapsed_ms);
    resized.image_write_to_file("resized.png").unwrap();
    Ok(resized.image_write_to_buffer(image_extension)?)
}

#[web::get("/resize/{encodedUrl}/")]
async fn resize(path: web::types::Path<(String)>, st: web::types::State<AppState>) -> impl web::Responder {
    let img_url = path.into_inner();
    println!("Path: {:?}", img_url);

    // decode base64
    let decoded = base64_url::decode(&img_url).unwrap();
    let url = std::str::from_utf8(&decoded).unwrap();

    // extract image extension and original image name
    let url_parts: Vec<&str> = url.split('/').collect();
    let file_name = url_parts.last().unwrap_or(&"resized.png");
    let file_parts: Vec<&str> = file_name.split('.').collect();
    let ext = format!(".{}", file_parts.last().unwrap_or(&"png"));

    let resized_image = resize_libvip(&st.http_client, url, &ext).await.unwrap();
    web::HttpResponse::Ok()
        .content_type(format!("image/{}", ext))
        .header("Content-Disposition", format!("attachment; filename={}", file_name))
        .body(resized_image)
}

struct AppState {
    http_client: Client,
    vips: VipsApp,
}

#[ntex::main]
async fn main() -> std::io::Result<()> {
    // let img_url = "https://display-events.adgreetz.com/sample.png";
    // load TLS keys
    // to create a self-signed temporary cert for testing:
    // `openssl req -x509 -newkey rsa:4096 -nodes -keyout key.pem -out cert.pem -days 365 -subj '/CN=localhost'`
    if std::env::var("RUST_LOG").is_err() {
        std::env::set_var("RUST_LOG", "info");
    }
    env_logger::init();

    // load ssl keys
    // let key_file = &mut BufReader::new(File::open("key.pem").unwrap());
    // let key = PrivateKey(rsa_private_keys(key_file).unwrap().remove(0));
    // let cert_file = &mut BufReader::new(File::open("cert.pem").unwrap());
    // let cert_chain = certs(cert_file)
    //     .unwrap()
    //     .iter()
    //     .map(|c| Certificate(c.to_vec()))
    //     .collect();
    // let config = ServerConfig::builder()
    //     .with_safe_defaults()
    //     .with_no_client_auth()
    //     .with_single_cert(cert_chain, key)
    //     .unwrap();

    web::HttpServer::new(|| {
        web::App::new().state(
            AppState {
                http_client: ClientBuilder::new().disable_timeout().finish(),
                vips: {let vips = VipsApp::new("Libvips", true).unwrap();
                    vips.concurrency_set(2);
                    vips},
            }
        )
            .service(resize)
    })
        // .bind_rustls(("0.0.0.0", 8080), config)?
        .bind("0.0.0.0:8080")?
        .run()
        .await

}

async fn download_image(http_client: &Client, url: &str) -> std::result::Result<Vec<u8>, reqwest::Error> {

    let mut response = http_client.get(url) // <- Create request builder
        .header("User-Agent", "ntex::web")
        .send()                             // <- Send http request
        .await.unwrap();
    let bytes = response.body().await.unwrap().to_vec();
    Ok(bytes)
}

Cargo.toml:

[package]
name = "image-resizer"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[profile.release]
lto = true

[dependencies]
env_logger = "0.10"
fast_image_resize = "2.7.3"
image = "0.24.6"
libvips = "1.5.1"
#tokio = { version = "1.28.1", features = ["macros", "rt-multi-thread"] }
base64-url = "2.0.0"
#ntex = { version = "0.6.7", features = ["tokio", "rustls"] }
ntex = {git = "https://github.com/ntex-rs/ntex", branch = "glommio-borrow-fix" , features = ["glommio", "rustls"] }

rustls = "0.20.8"
rustls-pemfile = "1.0.2"

I'm using vegeta for load testing: | vegeta attack -duration=2s -rate=100 -insecure -timeout=10s | vegeta report

The server panics:

   Finished release [optimized] target(s) in 0.11s
     Running `target/release/image-resizer`
[2023-05-30T13:05:11Z INFO  ntex::server::builder] Starting 2 workers
[2023-05-30T13:05:11Z INFO  ntex::server::builder] Starting "ntex-web-service-0.0.0.0:8080" service on 0.0.0.0:8080
[2023-05-30T13:05:11Z INFO  ntex::server::accept] Starting socket listener on 0.0.0.0:8080
thread 'ntex-rt:worker:1' panicked at 'already mutably borrowed: BorrowError', /root/.cargo/registry/src/github.com-1ecc6299db9ec823/ntex-glommio-0.2.3/src/io.rs:182:34
stack backtrace:
   0:     0x5585ba8e3a1d - std::backtrace_rs::backtrace::libunwind::trace::ha9053a9a07ca49cb
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0x5585ba8e3a1d - std::backtrace_rs::backtrace::trace_unsynchronized::h9c2852a457ad564e
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x5585ba8e3a1d - std::sys_common::backtrace::_print_fmt::h457936fbfaa0070f
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:65:5
   3:     0x5585ba8e3a1d - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h5779d7bf7f70cb0c
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x5585ba64fd9e - core::fmt::write::h5a4baaff1bcd3eb5
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/fmt/mod.rs:1232:17
   5:     0x5585ba8bd3e2 - std::io::Write::write_fmt::h4bc1f301cb9e9cce
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/io/mod.rs:1684:15
   6:     0x5585ba8e520f - std::sys_common::backtrace::_print::h5fcdc36060f177e8
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:47:5
   7:     0x5585ba8e520f - std::sys_common::backtrace::print::h54ca9458b876c8bf
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:34:9
   8:     0x5585ba8e4e0d - std::panicking::default_hook::{{closure}}::hbe471161c7664ed6
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:271:22
   9:     0x5585ba8e5e71 - std::panicking::default_hook::ha3500da57aa4ac4f
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:290:9
  10:     0x5585ba8e5e71 - std::panicking::rust_panic_with_hook::h50c09d000dc561d2
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:692:13
  11:     0x5585ba8e5934 - std::panicking::begin_panic_handler::{{closure}}::h9e2b2176e00e0d9c
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:583:13
  12:     0x5585ba8e5896 - std::sys_common::backtrace::__rust_end_short_backtrace::h5739b8e512c09d02
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:150:18
  13:     0x5585ba8e5881 - rust_begin_unwind
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
  14:     0x5585ba5ed672 - core::panicking::panic_fmt::hf33a1475b4dc5c3e
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
  15:     0x5585ba5ed992 - core::result::unwrap_failed::hdff5465d74574b44
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/result.rs:1750:5
  16:     0x5585ba7b3aef - <ntex_glommio::io::WriteTask as core::future::future::Future>::poll::{{closure}}::h513e49a87b7a9fe7
  17:     0x5585ba7b22f9 - <ntex_glommio::io::WriteTask as core::future::future::Future>::poll::h18b1f84fde9dadc2
  18:     0x5585ba7b2d7c - <ntex_glommio::io::WriteTask as core::future::future::Future>::poll::h18b1f84fde9dadc2
  19:     0x5585ba7aa536 - glommio::task::raw::RawTask<F,R,S>::run::ha5dc0e3d5ab443ad
  20:     0x5585ba67b9a1 - glommio::executor::LocalExecutor::run_task_queues::hb86087bd8ab27d9e
  21:     0x5585ba7f6f4e - std::sys_common::backtrace::__rust_begin_short_backtrace::h026df1cb73e762cc
  22:     0x5585ba7f0466 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h214ac7c93f77eb07
  23:     0x5585ba8e6bf5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h39990b24eedef2ab
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
  24:     0x5585ba8e6bf5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h01a027258444143b
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
  25:     0x5585ba8e6bf5 - std::sys::unix::thread::Thread::new::thread_start::ha4f1cdd9c25884ba
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys/unix/thread.rs:108:17
  26:     0x7fb38308f18a - <unknown>
  27:     0x7fb38311dbd0 - <unknown>
  28:                0x0 - <unknown>
Vips execution time: 1482 ms
Vips execution time: 1659 ms
thread 'ntex-rt:worker:0' panicked at 'already mutably borrowed: BorrowError', /root/.cargo/registry/src/github.com-1ecc6299db9ec823/ntex-glommio-0.2.3/src/io.rs:182:34
stack backtrace:
   0:     0x5585ba8e3a1d - std::backtrace_rs::backtrace::libunwind::trace::ha9053a9a07ca49cb
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0x5585ba8e3a1d - std::backtrace_rs::backtrace::trace_unsynchronized::h9c2852a457ad564e
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x5585ba8e3a1d - std::sys_common::backtrace::_print_fmt::h457936fbfaa0070f
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:65:5
   3:     0x5585ba8e3a1d - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h5779d7bf7f70cb0c
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x5585ba64fd9e - core::fmt::write::h5a4baaff1bcd3eb5
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/fmt/mod.rs:1232:17
   5:     0x5585ba8bd3e2 - std::io::Write::write_fmt::h4bc1f301cb9e9cce
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/io/mod.rs:1684:15
   6:     0x5585ba8e520f - std::sys_common::backtrace::_print::h5fcdc36060f177e8
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:47:5
   7:     0x5585ba8e520f - std::sys_common::backtrace::print::h54ca9458b876c8bf
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:34:9
   8:     0x5585ba8e4e0d - std::panicking::default_hook::{{closure}}::hbe471161c7664ed6
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:271:22
   9:     0x5585ba8e5e71 - std::panicking::default_hook::ha3500da57aa4ac4f
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:290:9
  10:     0x5585ba8e5e71 - std::panicking::rust_panic_with_hook::h50c09d000dc561d2
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:692:13
  11:     0x5585ba8e5934 - std::panicking::begin_panic_handler::{{closure}}::h9e2b2176e00e0d9c
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:583:13
  12:     0x5585ba8e5896 - std::sys_common::backtrace::__rust_end_short_backtrace::h5739b8e512c09d02
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:150:18
  13:     0x5585ba8e5881 - rust_begin_unwind
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
  14:     0x5585ba5ed672 - core::panicking::panic_fmt::hf33a1475b4dc5c3e
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
  15:     0x5585ba5ed992 - core::result::unwrap_failed::hdff5465d74574b44
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/result.rs:1750:5
  16:     0x5585ba7b3aef - <ntex_glommio::io::WriteTask as core::future::future::Future>::poll::{{closure}}::h513e49a87b7a9fe7
  17:     0x5585ba7b22f9 - <ntex_glommio::io::WriteTask as core::future::future::Future>::poll::h18b1f84fde9dadc2
  18:     0x5585ba7b2d7c - <ntex_glommio::io::WriteTask as core::future::future::Future>::poll::h18b1f84fde9dadc2
  19:     0x5585ba7aa536 - glommio::task::raw::RawTask<F,R,S>::run::ha5dc0e3d5ab443ad
  20:     0x5585ba67b9a1 - glommio::executor::LocalExecutor::run_task_queues::hb86087bd8ab27d9e
  21:     0x5585ba7f6f4e - std::sys_common::backtrace::__rust_begin_short_backtrace::h026df1cb73e762cc
  22:     0x5585ba7f0466 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h214ac7c93f77eb07
  23:     0x5585ba8e6bf5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h39990b24eedef2ab
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
  24:     0x5585ba8e6bf5 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h01a027258444143b
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
  25:     0x5585ba8e6bf5 - std::sys::unix::thread::Thread::new::thread_start::ha4f1cdd9c25884ba
                               at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys/unix/thread.rs:108:17
  26:     0x7fb38308f18a - <unknown>
  27:     0x7fb38311dbd0 - <unknown>
  28:                0x0 - <unknown>

[fafhrd91]

are you sure you use right branch? your last backtrace points to old code

[pionoor]

I mean i'm using: ntex = {git = "https://github.com/ntex-rs/ntex", branch = "glommio-borrow-fix" , features = ["glommio", "rustls"] } in my cargo. Let me clean and rebuild.

[pionoor]

I just double checked and cleaned and rerun, same error. Not sure why it is pointing to the old code.

[fafhrd91]

i think you need to override ntex-glommio only

[pionoor]

Can you please elaborate more?

[fafhrd91]

you need this entry to cargo.toml

[patch.crates-io]
ntex-glommio = {git = "https://github.com/ntex-rs/ntex", branch = "glommio-borrow-fix"}

[pionoor]

I did:

[dependencies]
anyhow = "1.0.71"
env_logger = "0.10"
fast_image_resize = "2.7.3"
image = "0.24.6"
libvips = "1.5.1"
base64-url = "2.0.0"
ntex = { version = "0.6.7", features = ["glommio", "rustls"] }

rustls = "0.20.8"
rustls-pemfile = "1.0.2"

[patch.crates-io]
ntex-glommio = {git = "https://github.com/ntex-rs/ntex", branch = "glommio-borrow-fix"}

It is not panicking anymore!! It looks like that was the fix!

[fafhrd91]

fixed in master

[pionoor]

Last question, what is the difference between using features = ["ntex-glommio"] vs features = ["glommio"]?

Which should I use?

[fafhrd91]

features = ["glommio"], ntex-glommio does not exists