uriahf / rtichoke

rtichoke - interactive visualizations for performance of predictive models
https://uriahf.github.io/rtichoke/
Other
11 stars 1 forks source link

Support zero variance when there are zero-variance predictions #83

Closed uriahf closed 1 year ago

uriahf commented 2 years ago

create_summary_report() does not work when there are zero-variance predictions.

This issue is similar to another issue about zero-variance outcomes: https://github.com/uriahf/rtichoke/issues/69

uriahf commented 1 year ago

These examples should work fine now

create_calibration_curve(
  probs = list(rep(0.23, 150)),
  reals = list(example_dat$outcome)
)

create_calibration_curve(
  probs = list(
    "First Model" = example_dat$estimated_probabilities,
    "Second Model" = rep(0.23, 150)
  ),
  reals = list(example_dat$outcome)
)

create_calibration_curve(
  probs = list(
    "train" = rep(0.23, 93),
    "test" = example_dat %>% dplyr::filter(type_of_set == "test") %>%
      dplyr::pull(estimated_probabilities)
  ),
  reals = list(
    "train" = example_dat %>% dplyr::filter(type_of_set == "train") %>%
      dplyr::pull(outcome),
    "test" = example_dat %>% dplyr::filter(type_of_set == "test") %>%
      dplyr::pull(outcome)
  )
)

These examples should work fine now

create_performance_table(
  probs = list(
    "Second Model" = rep(0.23, 150),
    "First Model" = example_dat$estimated_probabilities
  ),
  reals = list(example_dat$outcome), 
  stratified_by = "ppcr"
)

create_performance_table(
  probs = list(
    "Second Model" = rep(0.23, 150),
    "First Model" = example_dat$estimated_probabilities
  ),
  reals = list(example_dat$outcome)
) 

create_performance_table(
  probs = list(
    "train" = rep(0.23, 96),
    "test" = example_dat %>% dplyr::filter(type_of_set == "test") %>%
      dplyr::pull(estimated_probabilities)
  ),
  reals = list(
    "train" = example_dat %>% dplyr::filter(type_of_set == "train") %>%
      dplyr::pull(outcome),
    "test" = example_dat %>% dplyr::filter(type_of_set == "test") %>%
      dplyr::pull(outcome)
  ), 
  stratified_by = "ppcr"
)

These examples should work fine now

create_roc_curve(
  probs = list(rep(0.23, 150)),
  reals = list(example_dat$outcome),
  stratified_by = "ppcr"
)

create_roc_curve(
  probs = list(
    "First Model" = example_dat$estimated_probabilities,
    "Second Model" = rep(0.23, 150)
  ),
  reals = list(example_dat$outcome),
  stratified_by = "ppcr"
)

create_roc_curve(
  probs = list(
    "train" = rep(0.23, 93),
    "test" = example_dat %>% dplyr::filter(type_of_set == "test") %>%
      dplyr::pull(estimated_probabilities)
  ),
  reals = list(
    "train" = example_dat %>% dplyr::filter(type_of_set == "train") %>%
      dplyr::pull(outcome),
    "test" = example_dat %>% dplyr::filter(type_of_set == "test") %>%
      dplyr::pull(outcome)
  ),
  stratified_by = "ppcr"
)