Closed mvankessel-EMC closed 6 days ago
schuemie
commented
1 month ago Thanks! I think i found the problem: Eunomia has observation periods that start before 1970, but mktime() (referenced here) does not support dates before 1970, at least not on Windows.
Currently looking for a solution that does not involve implementing a full date computation stack c++ ;-)
mvankessel-EMC
commented
1 month ago Interresting. I did some digging yesterday, where I just added print statements to see where it exactly crashed. It seems that adding print statements here alters the behavior where it doesn't crash on bad allocation anymore. The sccsData object is empty, probably because of the time periods issue you described.
library(SelfControlledCaseSeries)
#> Loading required package: Cyclops
#> Loading required package: DatabaseConnector
#> Loading required package: Andromeda
#> Loading required package: dplyr
#>
#> Attaching package: 'dplyr'
#> The following objects are masked from 'package:stats':
#>
#> filter, lag
#> The following objects are masked from 'package:base':
#>
#> intersect, setdiff, setequal, unionlibrary(Eunomia)
connectionDetails <- getEunomiaConnectionDetails()
outputFolder <- file.path(tempdir(), "eunomia-results")
unlink(outputFolder, recursive = TRUE, force = TRUE)
cdmDatabaseSchema <- "main"
cohortDatabaseSchema <- "main"
options(sqlRenderTempEmulationSchema = NULL)
cdmVersion <- "5"
giBleed <- 77
diclofenac <- 1124300
cohortDefinitionSet <- PhenotypeLibrary::getPlCohortDefinitionSet(giBleed)
connection <- DatabaseConnector::connect(connectionDetails)
#> Connecting using SQLite driver
cohortTableNames <- CohortGenerator::getCohortTableNames(cohortTable = "cohort_table")
CohortGenerator::createCohortTables(connection = connection,
cohortDatabaseSchema = cohortDatabaseSchema,
cohortTableNames = cohortTableNames)
#> Creating cohort tables
#> - Created table main.cohort_table
#> - Created table main.cohort_table
#> - Created table main.cohort_table_inclusion
#> - Created table main.cohort_table_inclusion_result
#> - Created table main.cohort_table_inclusion_stats
#> - Created table main.cohort_table_summary_stats
#> - Created table main.cohort_table_censor_stats
#> Creating cohort tables took 0.13secs
counts <- CohortGenerator::generateCohortSet(connection = connection,
cdmDatabaseSchema = cdmDatabaseSchema,
cohortDatabaseSchema = cohortDatabaseSchema,
cohortTableNames = cohortTableNames,
cohortDefinitionSet = cohortDefinitionSet)
#> Initiating cluster consisting only of main thread
#> 1/1- Generating cohort: Gastrointestinal bleeding with inpatient admission (id = 77)
#> | | | 0% | |== | 3% | |===== | 6% | |======= | 10% | |========= | 13% | |=========== | 16% | |============== | 19% | |================ | 23% | |================== | 26% | |==================== | 29% | |======================= | 32% | |========================= | 35% | |=========================== | 39% | |============================= | 42% | |================================ | 45% | |================================== | 48% | |==================================== | 52% | |====================================== | 55% | |========================================= | 58% | |=========================================== | 61% | |============================================= | 65% | |=============================================== | 68% | |================================================== | 71% | |==================================================== | 74% | |====================================================== | 77% | |======================================================== | 81% | |=========================================================== | 84% | |============================================================= | 87% | |=============================================================== | 90% | |================================================================= | 94% | |==================================================================== | 97% | |======================================================================| 100%
#> Executing SQL took 0.0709 secs
#> Generating cohort set took 0.27 secs
DatabaseConnector::disconnect(connection)
sccsData <- getDbSccsData(connectionDetails = connectionDetails,
cdmDatabaseSchema = cdmDatabaseSchema,
outcomeDatabaseSchema = cohortDatabaseSchema,
outcomeTable = "cohort_table",
outcomeIds = giBleed,
exposureDatabaseSchema = cdmDatabaseSchema,
exposureTable = "drug_era",
exposureIds = diclofenac)
#> Connecting using SQLite driver
#> Inserting data took 0.0114 secs
#> Inserting data took 0.0282 secs
#> Selecting outcomes
#> | | | 0% | |======================= | 33% | |=============================================== | 67% | |======================================================================| 100%
#> Executing SQL took 0.00394 secs
#> Creating cases
#> | | | 0% | |======================= | 33% | |=============================================== | 67% | |======================================================================| 100%
#> Executing SQL took 0.00507 secs
#> Counting outcomes
#>
#> Creating eras
#> | | | 0% | |======== | 11% | |================ | 22% | |======================= | 33% | |=============================== | 44% | |======================================= | 56% | |=============================================== | 67% | |====================================================== | 78% | |============================================================== | 89% | |======================================================================| 100%
#> Executing SQL took 0.0214 secs
#> Fetching data from server
#> Fetched 391 cases from server
#> Getting SCCS data from server took 0.521 secs
studyPop <- createStudyPopulation(sccsData = sccsData,
outcomeId = giBleed,
firstOutcomeOnly = FALSE,
naivePeriod = 180)
covarDiclofenac <- createEraCovariateSettings(label = "Exposure of interest",
includeEraIds = diclofenac,
start = 0,
end = 0,
endAnchor = "era end")
sccsIntervalData <- createSccsIntervalData(
studyPopulation = studyPop,
sccsData = sccsData,
eraCovariateSettings = covarDiclofenac,
seasonalityCovariateSettings = createSeasonalityCovariateSettings(),
calendarTimeCovariateSettings = createCalendarTimeCovariateSettings()
)
#> Converting person data to SCCS intervals. This might take a while.
#> Warning: ORDER BY is ignored in subqueries without LIMIT
#> ℹ Do you need to move arrange() later in the pipeline or use window_order() instead?
#> | | | 0% | |======================================================================| 100%
#> Warning in createSccsIntervalData(studyPopulation = studyPop, sccsData =
#> sccsData, : Conversion resulted in empty data set. Perhaps no one with the
#> outcome had any exposure of interest?
#> Generating SCCS interval data took 0.269 secs
model <- fitSccsModel(sccsIntervalData)
#> Fitting SCCS model
#> Fitting the model took 0.03 secs
#> Model fitting status is: Could not estimate because there was no data
model
#> SccsModel object
#>
#> Outcome ID: 77
#>
#> Outcome count:
#> outcomeSubjects outcomeEvents outcomeObsPeriods observedDays
#> 77 0 0 0 0
#>
#> Status: Could not estimate because there was no dataCreated on 2024-06-20 with reprex v2.1.0
schuemie
commented
1 month ago Ok, should be fixed in develop now. (I did end up implementing my own date manipulation functions. The alternatives were requiring C++ 20 or the Boost library. This seemed the cleanest.)
The model doesn't converge, just because Eunomia was never design for SCCS.
mvankessel-EMC
commented
1 month ago Ok, should be fixed in develop now. (I did end up implementing my own date manipulation functions. The alternatives were requiring C++ 20 or the Boost library. This seemed the cleanest.)
It seems to work, thanks!
The model doesn't converge, just because Eunomia was never design for SCCS.
That makes sense.
library(SelfControlledCaseSeries)
#> Loading required package: Cyclops
#> Loading required package: DatabaseConnector
#> Loading required package: Andromeda
#> Loading required package: dplyr
#>
#> Attaching package: 'dplyr'
#> The following objects are masked from 'package:stats':
#>
#> filter, lag
#> The following objects are masked from 'package:base':
#>
#> intersect, setdiff, setequal, unionlibrary(Eunomia)
connectionDetails <- getEunomiaConnectionDetails()
outputFolder <- file.path(tempdir(), "eunomia-results")
unlink(outputFolder, recursive = TRUE, force = TRUE)
cdmDatabaseSchema <- "main"
cohortDatabaseSchema <- "main"
options(sqlRenderTempEmulationSchema = NULL)
cdmVersion <- "5"
giBleed <- 77
diclofenac <- 1124300
cohortDefinitionSet <- PhenotypeLibrary::getPlCohortDefinitionSet(giBleed)
connection <- DatabaseConnector::connect(connectionDetails)
#> Connecting using SQLite driver
cohortTableNames <- CohortGenerator::getCohortTableNames(cohortTable = "cohort_table")
CohortGenerator::createCohortTables(connection = connection,
cohortDatabaseSchema = cohortDatabaseSchema,
cohortTableNames = cohortTableNames)
#> Creating cohort tables
#> - Created table main.cohort_table
#> - Created table main.cohort_table
#> - Created table main.cohort_table_inclusion
#> - Created table main.cohort_table_inclusion_result
#> - Created table main.cohort_table_inclusion_stats
#> - Created table main.cohort_table_summary_stats
#> - Created table main.cohort_table_censor_stats
#> Creating cohort tables took 0.13secs
counts <- CohortGenerator::generateCohortSet(connection = connection,
cdmDatabaseSchema = cdmDatabaseSchema,
cohortDatabaseSchema = cohortDatabaseSchema,
cohortTableNames = cohortTableNames,
cohortDefinitionSet = cohortDefinitionSet)
#> Initiating cluster consisting only of main thread
#> 1/1- Generating cohort: Gastrointestinal bleeding with inpatient admission (id = 77)
#> | | | 0% | |== | 3% | |===== | 6% | |======= | 10% | |========= | 13% | |=========== | 16% | |============== | 19% | |================ | 23% | |================== | 26% | |==================== | 29% | |======================= | 32% | |========================= | 35% | |=========================== | 39% | |============================= | 42% | |================================ | 45% | |================================== | 48% | |==================================== | 52% | |====================================== | 55% | |========================================= | 58% | |=========================================== | 61% | |============================================= | 65% | |=============================================== | 68% | |================================================== | 71% | |==================================================== | 74% | |====================================================== | 77% | |======================================================== | 81% | |=========================================================== | 84% | |============================================================= | 87% | |=============================================================== | 90% | |================================================================= | 94% | |==================================================================== | 97% | |======================================================================| 100%
#> Executing SQL took 0.079 secs
#> Generating cohort set took 0.28 secs
DatabaseConnector::disconnect(connection)
sccsData <- getDbSccsData(connectionDetails = connectionDetails,
cdmDatabaseSchema = cdmDatabaseSchema,
outcomeDatabaseSchema = cohortDatabaseSchema,
outcomeTable = "cohort_table",
outcomeIds = giBleed,
exposureDatabaseSchema = cdmDatabaseSchema,
exposureTable = "drug_era",
exposureIds = diclofenac)
#> Connecting using SQLite driver
#> Inserting data took 0.0118 secs
#> Inserting data took 0.028 secs
#> Selecting outcomes
#> | | | 0% | |======================= | 33% | |=============================================== | 67% | |======================================================================| 100%
#> Executing SQL took 0.00387 secs
#> Creating cases
#> | | | 0% | |======================= | 33% | |=============================================== | 67% | |======================================================================| 100%
#> Executing SQL took 0.00511 secs
#> Counting outcomes
#>
#> Creating eras
#> | | | 0% | |======== | 11% | |================ | 22% | |======================= | 33% | |=============================== | 44% | |======================================= | 56% | |=============================================== | 67% | |====================================================== | 78% | |============================================================== | 89% | |======================================================================| 100%
#> Executing SQL took 0.0218 secs
#> Fetching data from server
#> Fetched 391 cases from server
#> Getting SCCS data from server took 0.514 secs
studyPop <- createStudyPopulation(sccsData = sccsData,
outcomeId = giBleed,
firstOutcomeOnly = FALSE,
naivePeriod = 180)
covarDiclofenac <- createEraCovariateSettings(label = "Exposure of interest",
includeEraIds = diclofenac,
start = 0,
end = 0,
endAnchor = "era end")
sccsIntervalData <- createSccsIntervalData(
studyPopulation = studyPop,
sccsData = sccsData,
eraCovariateSettings = covarDiclofenac,
seasonalityCovariateSettings = createSeasonalityCovariateSettings(),
calendarTimeCovariateSettings = createCalendarTimeCovariateSettings()
)
#> Converting person data to SCCS intervals. This might take a while.
#> | | | 0% | |======================================================================| 100%
#> Generating SCCS interval data took 1.7 secs
model <- fitSccsModel(sccsIntervalData)
#> Fitting SCCS model
#> Using prior: None
#>
#> Warning: problem is ill-conditioned for this choice of
#> prior (None) or
#> initial bounding box (2)
#> Enforcing convergence!
#>
#> Warning: problem is ill-conditioned for this choice of
#> prior (None) or
#> initial bounding box (0.2)
#> Enforcing convergence!
#>
#> Warning: problem is ill-conditioned for this choice of
#> prior (None) or
#> initial bounding box (0.02)
#> Enforcing convergence!
#>
#> Warning: problem is ill-conditioned for this choice of
#> prior (None) or
#> initial bounding box (0.002)
#> Enforcing convergence!
#>
#> Warning: problem is ill-conditioned for this choice of
#> prior (None) or
#> initial bounding box (0.0002)
#> Enforcing convergence!
#> Fitting the model took 5.34 secs
#> Model fitting status is: ILLCONDITIONED
model
#> SccsModel object
#>
#> Outcome ID: 77
#>
#> Outcome count:
#> outcomeSubjects outcomeEvents outcomeObsPeriods observedDays
#> 77 391 391 391 8327356
#>
#> Status: ILLCONDITIONEDCreated on 2024-06-20 with reprex v2.1.0
schuemie
commented
1 month ago I take that back. It does converge, but I had some legacy code in the way. Latest version in develop should work (and converge)
When running sccsAnalysis4 (I haven't tried 2, 3 or 5) from the MultipleAnalyses vignette, I run into a
std::bad_allocerror from C++ when creating the Interval Data (scroll to the side).I'm running on a machine with 32gb of RAM, and a rather small dataset from Eunomia, with 391 case. So I doubt it's a memory issue.
With every tick of the progress bar, it throws the same
std::bad_allocerror from a different thread, with a differentincludeEraId.Disabling both seasonality and calendar time covariates seems to work, but then obviously don't include those covariates.
In the Reprex below I isolate analysis4 according to the singular analysis vignette, running into the same issue:
Reprex isolated
Created on 2024-06-13 with reprex v2.1.0
Session info
``` r sessioninfo::session_info() #> ─ Session info ─────────────────────────────────────────────────────────────── #> setting value #> version R version 4.4.0 (2024-04-24 ucrt) #> os Windows 11 x64 (build 22631) #> system x86_64, mingw32 #> ui RTerm #> language (EN) #> collate Dutch_Netherlands.utf8 #> ctype Dutch_Netherlands.utf8 #> tz Europe/Amsterdam #> date 2024-06-13 #> pandoc 3.1.11 @ C:/Program Files/RStudio/resources/app/bin/quarto/bin/tools/ (via rmarkdown) #> #> ─ Packages ─────────────────────────────────────────────────────────────────── #> ! package * version date (UTC) lib source #> Andromeda * 0.6.6 2024-03-21 [1] CRAN (R 4.4.0) #> backports 1.5.0 2024-05-23 [1] CRAN (R 4.4.0) #> bit 4.0.5 2022-11-15 [1] CRAN (R 4.4.0) #> bit64 4.0.5 2020-08-30 [1] CRAN (R 4.4.0) #> blob 1.2.4 2023-03-17 [1] CRAN (R 4.4.0) #> cachem 1.1.0 2024-05-16 [1] CRAN (R 4.4.0) #> checkmate 2.3.1 2023-12-04 [1] CRAN (R 4.4.0) #> cli 3.6.2 2023-12-11 [1] CRAN (R 4.4.0) #> CohortGenerator 0.9.0 2024-06-01 [1] Github (ohdsi/CohortGenerator@e3efad6) #> crayon 1.5.2 2022-09-29 [1] CRAN (R 4.4.0) #> Cyclops * 3.4.1 2024-06-06 [1] CRAN (R 4.4.0) #> DatabaseConnector * 6.3.2 2023-12-11 [1] CRAN (R 4.4.0) #> DBI 1.2.3 2024-06-02 [1] CRAN (R 4.4.0) #> dbplyr 2.5.0 2024-03-19 [1] CRAN (R 4.4.0) #> digest 0.6.35 2024-03-11 [1] CRAN (R 4.4.0) #> dplyr * 1.1.4 2023-11-17 [1] CRAN (R 4.4.0) #> Eunomia * 2.0.0 2024-05-30 [1] Github (OHDSI/Eunomia@f016f27) #> evaluate 0.23 2023-11-01 [1] CRAN (R 4.4.0) #> fansi 1.0.6 2023-12-08 [1] CRAN (R 4.4.0) #> fastmap 1.2.0 2024-05-15 [1] CRAN (R 4.4.0) #> fs 1.6.4 2024-04-25 [1] CRAN (R 4.4.0) #> generics 0.1.3 2022-07-05 [1] CRAN (R 4.4.0) #> glue 1.7.0 2024-01-09 [1] CRAN (R 4.4.0) #> hms 1.1.3 2023-03-21 [1] CRAN (R 4.4.0) #> htmltools 0.5.8.1 2024-04-04 [1] CRAN (R 4.4.0) #> knitr 1.47 2024-05-29 [1] CRAN (R 4.4.0) #> lattice 0.22-6 2024-03-20 [1] CRAN (R 4.4.0) #> lifecycle 1.0.4 2023-11-07 [1] CRAN (R 4.4.0) #> lubridate 1.9.3 2023-09-27 [1] CRAN (R 4.4.0) #> magrittr 2.0.3 2022-03-30 [1] CRAN (R 4.4.0) #> Matrix 1.7-0 2024-03-22 [1] CRAN (R 4.4.0) #> memoise 2.0.1 2021-11-26 [1] CRAN (R 4.4.0) #> ParallelLogger 3.3.0 2023-08-22 [1] CRAN (R 4.4.0) #> PhenotypeLibrary 3.32.0 2024-05-28 [1] Github (OHDSI/PhenotypeLibrary@bccdd87) #> pillar 1.9.0 2023-03-22 [1] CRAN (R 4.4.0) #> pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.4.0) #> purrr 1.0.2 2023-08-10 [1] CRAN (R 4.4.0) #> R.cache 0.16.0 2022-07-21 [1] CRAN (R 4.4.0) #> R.methodsS3 1.8.2 2022-06-13 [1] CRAN (R 4.4.0) #> R.oo 1.26.0 2024-01-24 [1] CRAN (R 4.4.0) #> R.utils 2.12.3 2023-11-18 [1] CRAN (R 4.4.0) #> R6 2.5.1 2021-08-19 [1] CRAN (R 4.4.0) #> Rcpp 1.0.12 2024-01-09 [1] CRAN (R 4.4.0) #> readr 2.1.5 2024-01-10 [1] CRAN (R 4.4.0) #> reprex 2.1.0 2024-01-11 [1] CRAN (R 4.4.0) #> D rJava 1.0-11 2024-01-26 [1] CRAN (R 4.4.0) #> rlang 1.1.4 2024-06-04 [1] CRAN (R 4.4.0) #> rmarkdown 2.27 2024-05-17 [1] CRAN (R 4.4.0) #> RSQLite 2.3.7 2024-05-27 [1] CRAN (R 4.4.0) #> rstudioapi 0.16.0 2024-03-24 [1] CRAN (R 4.4.0) #> SelfControlledCaseSeries * 5.2.2 2024-06-13 [1] Github (OHDSI/SelfControlledCaseSeries@67b26f6) #> sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.4.0) #> SqlRender 1.18.0 2024-05-30 [1] CRAN (R 4.4.0) #> styler 1.10.3 2024-04-07 [1] CRAN (R 4.4.0) #> survival 3.5-8 2024-02-14 [1] CRAN (R 4.4.0) #> tibble 3.2.1 2023-03-20 [1] CRAN (R 4.4.0) #> tidyselect 1.2.1 2024-03-11 [1] CRAN (R 4.4.0) #> timechange 0.3.0 2024-01-18 [1] CRAN (R 4.4.0) #> tzdb 0.4.0 2023-05-12 [1] CRAN (R 4.4.0) #> utf8 1.2.4 2023-10-22 [1] CRAN (R 4.4.0) #> vctrs 0.6.5 2023-12-01 [1] CRAN (R 4.4.0) #> vroom 1.6.5 2023-12-05 [1] CRAN (R 4.4.0) #> withr 3.0.0 2024-01-16 [1] CRAN (R 4.4.0) #> xfun 0.44 2024-05-15 [1] CRAN (R 4.4.0) #> yaml 2.3.8 2023-12-11 [1] CRAN (R 4.4.0) #> #> [1] C:/R/R-4.4.0/library #> #> D ── DLL MD5 mismatch, broken installation. #> #> ────────────────────────────────────────────────────────────────────────────── ```