grimbough / FITfileR

R package for reading data from FIT files using only native R code, rather than relying on external libraries.
https://msmith.de/FITfileR
50 stars 13 forks source link

Can't read fit files #1

Closed muschellij2 closed 4 years ago

muschellij2 commented 5 years ago

Can't read fit files 2019-08-05.zip

Reproducible example:

library(fitFileR)
destfile = tempfile(fileext = ".zip")
url = "https://github.com/grimbough/fitFileR/files/3473420/2019-08-05.zip"
dl = download.file(url = url, destfile = destfile)
tdir = tempfile()
dir.create(tdir)
files = unzip(destfile, exdir = tdir)
fileName = files[1]

readFitFile(fileName)
#> Error in .readRecordHeader(con): Compressed time stamp header not currently supported
sapply(files, readFitFile)
#> Error in .readRecordHeader(con): Compressed time stamp header not currently supported

Created on 2019-08-06 by the reprex package (v0.3.0)

Session info ``` r devtools::session_info() #> ─ Session info ────────────────────────────────────────────────────────── #> setting value #> version R version 3.6.0 (2019-04-26) #> os macOS Mojave 10.14.6 #> system x86_64, darwin15.6.0 #> ui X11 #> language (EN) #> collate en_US.UTF-8 #> ctype en_US.UTF-8 #> tz America/New_York #> date 2019-08-06 #> #> ─ Packages ────────────────────────────────────────────────────────────── #> package * version date lib source #> assertthat 0.2.1 2019-03-21 [1] CRAN (R 3.6.0) #> backports 1.1.4 2019-04-10 [1] CRAN (R 3.6.0) #> callr 3.3.0 2019-07-04 [1] CRAN (R 3.6.0) #> cli 1.1.0 2019-03-19 [1] CRAN (R 3.6.0) #> crayon 1.3.4 2017-09-16 [1] CRAN (R 3.6.0) #> desc 1.2.0 2019-07-10 [1] Github (muschellij2/desc@b0c374f) #> devtools 2.1.0 2019-07-06 [1] CRAN (R 3.6.0) #> digest 0.6.20 2019-07-04 [1] CRAN (R 3.6.0) #> dplyr 0.8.3 2019-07-04 [1] CRAN (R 3.6.0) #> evaluate 0.14 2019-05-28 [1] CRAN (R 3.6.0) #> fitFileR * 0.0.3 2019-08-06 [1] Github (grimbough/fitFileR@1bebfda) #> fs 1.3.1 2019-05-06 [1] CRAN (R 3.6.0) #> glue 1.3.1 2019-03-12 [1] CRAN (R 3.6.0) #> highr 0.8 2019-03-20 [1] CRAN (R 3.6.0) #> htmltools 0.3.6 2017-04-28 [1] CRAN (R 3.6.0) #> knitr 1.23 2019-05-18 [1] CRAN (R 3.6.0) #> magrittr 1.5 2014-11-22 [1] CRAN (R 3.6.0) #> memoise 1.1.0 2017-04-21 [1] CRAN (R 3.6.0) #> pillar 1.4.2 2019-06-29 [1] CRAN (R 3.6.0) #> pkgbuild 1.0.3 2019-03-20 [1] CRAN (R 3.6.0) #> pkgconfig 2.0.2 2018-08-16 [1] CRAN (R 3.6.0) #> pkgload 1.0.2 2018-10-29 [1] CRAN (R 3.6.0) #> prettyunits 1.0.2 2015-07-13 [1] CRAN (R 3.6.0) #> processx 3.4.0 2019-07-03 [1] CRAN (R 3.6.0) #> ps 1.3.0 2018-12-21 [1] CRAN (R 3.6.0) #> purrr 0.3.2 2019-03-15 [1] CRAN (R 3.6.0) #> R6 2.4.0 2019-02-14 [1] CRAN (R 3.6.0) #> Rcpp 1.0.2 2019-07-25 [1] CRAN (R 3.6.0) #> remotes 2.1.0 2019-06-24 [1] CRAN (R 3.6.0) #> rlang 0.4.0 2019-06-25 [1] CRAN (R 3.6.0) #> rmarkdown 1.13 2019-05-22 [1] CRAN (R 3.6.0) #> rprojroot 1.3-2 2018-01-03 [1] CRAN (R 3.6.0) #> sessioninfo 1.1.1 2018-11-05 [1] CRAN (R 3.6.0) #> stringi 1.4.3 2019-03-12 [1] CRAN (R 3.6.0) #> stringr 1.4.0 2019-02-10 [1] CRAN (R 3.6.0) #> testthat 2.1.1 2019-04-23 [1] CRAN (R 3.6.0) #> tibble 2.1.3 2019-06-06 [1] CRAN (R 3.6.0) #> tidyselect 0.2.5 2018-10-11 [1] CRAN (R 3.6.0) #> usethis 1.5.1 2019-07-04 [1] CRAN (R 3.6.0) #> withr 2.1.2 2018-03-15 [1] CRAN (R 3.6.0) #> xfun 0.8 2019-06-25 [1] CRAN (R 3.6.0) #> yaml 2.2.0 2018-07-25 [1] CRAN (R 3.6.0) #> #> [1] /Library/Frameworks/R.framework/Versions/3.6/Resources/library ```
muschellij2 commented 5 years ago

Seems to be the 23rd record, which I'm not sure is relevant, but looks like developer_data = TRUE:

library(fitFileR)
destfile = tempfile(fileext = ".zip")
url = "https://github.com/grimbough/fitFileR/files/3473420/2019-08-05.zip"
dl = download.file(url = url, destfile = destfile)
tdir = tempfile()
dir.create(tdir)
files = unzip(destfile, exdir = tdir)
fileName = files[1]

con = file(fileName, "rb")

data("data_type_lookup", package = "fitFileR")
file_header <- fitFileR:::.readFileHeader(con)

plmt <- "-1";
pseudoMessageTab <- NULL
prev_lmt <- "0"
defs <- list()
defs_count <- list()

for (i in 1:23) {

  print(paste0("index is ", seek(con, where = NA)))
  if (seek(con, where = NA) < (file_header$data_size + 14)) {
    print(i)
  } else { 
    break
  }
  record_header <-  fitFileR:::.readRecordHeader(con)
  lmt <- as.character(record_header$local_message_type)
  if(record_header$message_type == "definition") {

    print('definition')
    if(lmt == prev_lmt) {
      plmt <- as.character(as.integer(plmt) + 1)
    } else {
      plmt <- lmt
    }
    pseudoMessageTab <- rbind(pseudoMessageTab, c(lmt, plmt))
    prev_lmt <- lmt

    message <- fitFileR:::.readMessage.definition(con, devFields = record_header$developer_data)
    # print(message)
    defs[[ plmt ]] <- message$message
    defs_count[[ plmt ]] <- 0

  } else if(record_header$message_type == "data") {

    print('data')

    defIdx <- pseudoMessageTab[ max(which(pseudoMessageTab[,1] == lmt)), 2]
    message <- fitFileR:::.scanMessage.data(con, defs[[ defIdx ]])
    # print(message)
    defs_count[[ defIdx ]] <- defs_count[[ defIdx ]] + 1

  } else {
    stop("unknown message type")
  }
}
#> [1] "index is 14"
#> [1] 1
#> [1] "definition"
#> [1] "index is 41"
#> [1] 2
#> [1] "data"
#> [1] "index is 59"
#> [1] 3
#> [1] "definition"
#> [1] "index is 80"
#> [1] 4
#> [1] "data"
#> [1] "index is 95"
#> [1] 5
#> [1] "definition"
#> [1] "index is 113"
#> [1] 6
#> [1] "data"
#> [1] "index is 125"
#> [1] 7
#> [1] "definition"
#> [1] "index is 137"
#> [1] 8
#> [1] "data"
#> [1] "index is 143"
#> [1] 9
#> [1] "data"
#> [1] "index is 149"
#> [1] 10
#> [1] "data"
#> [1] "index is 155"
#> [1] 11
#> [1] "data"
#> [1] "index is 161"
#> [1] 12
#> [1] "data"
#> [1] "index is 167"
#> [1] 13
#> [1] "data"
#> [1] "index is 173"
#> [1] 14
#> [1] "data"
#> [1] "index is 179"
#> [1] 15
#> [1] "data"
#> [1] "index is 185"
#> [1] 16
#> [1] "data"
#> [1] "index is 191"
#> [1] 17
#> [1] "data"
#> [1] "index is 197"
#> [1] 18
#> [1] "data"
#> [1] "index is 203"
#> [1] 19
#> [1] "data"
#> [1] "index is 209"
#> [1] 20
#> [1] "data"
#> [1] "index is 215"
#> [1] 21
#> [1] "definition"
#> [1] "index is 224"
#> [1] 22
#> [1] "data"
#> [1] "index is 225"
#> [1] 23
#> Error in fitFileR:::.readRecordHeader(con): Compressed time stamp header not currently supported

close(con)

Created on 2019-08-06 by the reprex package (v0.3.0)

Session info ``` r devtools::session_info() #> ─ Session info ────────────────────────────────────────────────────────── #> setting value #> version R version 3.6.0 (2019-04-26) #> os macOS Mojave 10.14.6 #> system x86_64, darwin15.6.0 #> ui X11 #> language (EN) #> collate en_US.UTF-8 #> ctype en_US.UTF-8 #> tz America/New_York #> date 2019-08-06 #> #> ─ Packages ────────────────────────────────────────────────────────────── #> package * version date lib source #> assertthat 0.2.1 2019-03-21 [1] CRAN (R 3.6.0) #> backports 1.1.4 2019-04-10 [1] CRAN (R 3.6.0) #> callr 3.3.0 2019-07-04 [1] CRAN (R 3.6.0) #> cli 1.1.0 2019-03-19 [1] CRAN (R 3.6.0) #> crayon 1.3.4 2017-09-16 [1] CRAN (R 3.6.0) #> desc 1.2.0 2019-07-10 [1] Github (muschellij2/desc@b0c374f) #> devtools 2.1.0 2019-07-06 [1] CRAN (R 3.6.0) #> digest 0.6.20 2019-07-04 [1] CRAN (R 3.6.0) #> dplyr 0.8.3 2019-07-04 [1] CRAN (R 3.6.0) #> evaluate 0.14 2019-05-28 [1] CRAN (R 3.6.0) #> fitFileR * 0.0.3 2019-08-06 [1] Github (grimbough/fitFileR@1bebfda) #> fs 1.3.1 2019-05-06 [1] CRAN (R 3.6.0) #> glue 1.3.1 2019-03-12 [1] CRAN (R 3.6.0) #> highr 0.8 2019-03-20 [1] CRAN (R 3.6.0) #> htmltools 0.3.6 2017-04-28 [1] CRAN (R 3.6.0) #> knitr 1.23 2019-05-18 [1] CRAN (R 3.6.0) #> magrittr 1.5 2014-11-22 [1] CRAN (R 3.6.0) #> memoise 1.1.0 2017-04-21 [1] CRAN (R 3.6.0) #> pillar 1.4.2 2019-06-29 [1] CRAN (R 3.6.0) #> pkgbuild 1.0.3 2019-03-20 [1] CRAN (R 3.6.0) #> pkgconfig 2.0.2 2018-08-16 [1] CRAN (R 3.6.0) #> pkgload 1.0.2 2018-10-29 [1] CRAN (R 3.6.0) #> prettyunits 1.0.2 2015-07-13 [1] CRAN (R 3.6.0) #> processx 3.4.0 2019-07-03 [1] CRAN (R 3.6.0) #> ps 1.3.0 2018-12-21 [1] CRAN (R 3.6.0) #> purrr 0.3.2 2019-03-15 [1] CRAN (R 3.6.0) #> R6 2.4.0 2019-02-14 [1] CRAN (R 3.6.0) #> Rcpp 1.0.2 2019-07-25 [1] CRAN (R 3.6.0) #> remotes 2.1.0 2019-06-24 [1] CRAN (R 3.6.0) #> rlang 0.4.0 2019-06-25 [1] CRAN (R 3.6.0) #> rmarkdown 1.13 2019-05-22 [1] CRAN (R 3.6.0) #> rprojroot 1.3-2 2018-01-03 [1] CRAN (R 3.6.0) #> sessioninfo 1.1.1 2018-11-05 [1] CRAN (R 3.6.0) #> stringi 1.4.3 2019-03-12 [1] CRAN (R 3.6.0) #> stringr 1.4.0 2019-02-10 [1] CRAN (R 3.6.0) #> testthat 2.1.1 2019-04-23 [1] CRAN (R 3.6.0) #> tibble 2.1.3 2019-06-06 [1] CRAN (R 3.6.0) #> tidyselect 0.2.5 2018-10-11 [1] CRAN (R 3.6.0) #> usethis 1.5.1 2019-07-04 [1] CRAN (R 3.6.0) #> withr 2.1.2 2018-03-15 [1] CRAN (R 3.6.0) #> xfun 0.8 2019-06-25 [1] CRAN (R 3.6.0) #> yaml 2.2.0 2018-07-25 [1] CRAN (R 3.6.0) #> #> [1] /Library/Frameworks/R.framework/Versions/3.6/Resources/library ```
grimbough commented 4 years ago

This was patched a while ago and you should be able to read those files now e.g.

library(fitFileR)
destfile = tempfile(fileext = ".zip")
url = "https://github.com/grimbough/fitFileR/files/3473420/2019-08-05.zip"
dl = download.file(url = url, destfile = destfile)
tdir = tempfile()
dir.create(tdir)
files = unzip(destfile, exdir = tdir)

fit_list <- lapply(files, readFitFile)
lapply(fit_list, names)
#> [[1]]
#> [1] "file_id"     "device_info"
#> 
#> [[2]]
#> [1] "file_id"     "device_info"
#> 
#> [[3]]
#> [1] "file_id"      "file_creator" "device_info" 
#> 
#> [[4]]
#> [1] "file_id"         "device_info"     "software"        "monitoring_info"
#> [5] "monitoring"      "ohr_settings"    "stress_level"    "event"          
#> 
#> [[5]]
#> [1] "file_id"         "device_info"     "software"        "monitoring_info"
#> [5] "monitoring"      "ohr_settings"    "stress_level"    "event"
fit_list[[5]][['event']]
#> # A tibble: 150 x 4
#>    timestamp           data16 event event_type
#>    <dttm>               <int> <chr> <chr>     
#>  1 2019-08-05 06:29:00     80 <NA>  marker    
#>  2 2019-08-05 06:30:00      1 <NA>  marker    
#>  3 2019-08-05 11:07:00    277 <NA>  marker    
#>  4 2019-08-05 11:10:00      3 <NA>  marker    
#>  5 2019-08-05 11:56:00     46 <NA>  marker    
#>  6 2019-08-05 11:57:00      1 <NA>  marker    
#>  7 2019-08-05 15:23:00    206 <NA>  marker    
#>  8 2019-08-05 15:28:00      5 <NA>  marker    
#>  9 2019-08-05 15:29:00      1 <NA>  marker    
#> 10 2019-08-05 15:30:00      1 <NA>  marker    
#> # … with 140 more rows