`Config::default().with_color(false)` not disable colors

[mamcx]

Despite setting this to false, the colors are shown when added to labels:

pub fn build_report(
    named: String,
    err: &ErrorParser,
) -> Report<(String, Range<usize>)> {
    let mut colors = ColorGenerator::new();

    // Generate some colours for each of our elements
    let primary = colors.next();
    // let secondary = colors.next();
    let code = err.error_code();
    let config = Config::default().with_color(false);
    let diagnostic = Report::build(ReportKind::Error, named.clone(), 0)
        .with_code(code as usize)
        .with_config(config);

    match err {
        ErrorParser::ScalarParse { span, kind, msg } => diagnostic
            .with_message(msg)
            .with_label(
                Label::new((named, span.range()))
                    .with_message(msg)
                    .with_color(primary), <-- THIS CAUSE IT TO SHOW COLORS
            )
            .with_note(format!("Parsing value of type: {kind:?}"))
            .finish(),
        _ => {
            todo!()
        }
    }
}